NBC News / robots.txt
← back to nbcnews.com · change detected 8d ago (2026-07-27T21:55:10Z) · modified +110 −0
What changed
- Blocked user-agent ‘AhrefsBot’ entirely
- Blocked user-agent ‘Archive.org_bot’ entirely
- Blocked user-agent ‘Brightbot 1.0’ entirely
- Blocked user-agent ‘ByteSpider Desktop’ entirely
- Blocked user-agent ‘ExaBot’ entirely
- Blocked user-agent ‘Petalbot’ entirely
- Blocked user-agent ‘QuillBot’ entirely
- Blocked user-agent ‘quillbot.com’ entirely
- Blocked user-agent ‘ShapBot’ entirely
- Blocked user-agent ‘ViennaTinyBot’ entirely
- 🤖 AIBlocked YouBot (You.com) entirely
Diff old (2026-07-13T22:52:30Z) → new (2026-07-27T21:55:10Z)
| @@ -16,6 +16,18 @@ | ||
| 16 | 16 | Allow: /*?*canonicalCard= |
| 17 | 17 | |
| 18 | 18 | # Disallow Bots (Allow /select only) |
| 19 | + User-agent: AhrefsBot | |
| 20 | + Disallow: / | |
| 21 | + Allow: /select | |
| 22 | + Allow: /select/ | |
| 23 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 24 | + Allow: /subscribe/supplemental-terms | |
| 25 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 26 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 27 | + Allow: /tips | |
| 28 | + Allow: /tips/ | |
| 29 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 30 | + | |
| 19 | 31 | User-agent: Amazonbot |
| 20 | 32 | User-agent: AmazonBot |
| 21 | 33 | Disallow: / |
| @@ -53,6 +65,18 @@ | ||
| 53 | 65 | Allow: /tips/ |
| 54 | 66 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 55 | 67 | |
| 68 | + User-agent: Archive.org_bot | |
| 69 | + Disallow: / | |
| 70 | + Allow: /select | |
| 71 | + Allow: /select/ | |
| 72 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 73 | + Allow: /subscribe/supplemental-terms | |
| 74 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 75 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 76 | + Allow: /tips | |
| 77 | + Allow: /tips/ | |
| 78 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 79 | + | |
| 56 | 80 | User-agent: AwarioRssBot |
| 57 | 81 | User-agent: AwarioSmartBot |
| 58 | 82 | Disallow: / |
| @@ -66,7 +90,20 @@ | ||
| 66 | 90 | Allow: /tips/ |
| 67 | 91 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 68 | 92 | |
| 93 | + User-agent: Brightbot 1.0 | |
| 94 | + Disallow: / | |
| 95 | + Allow: /select | |
| 96 | + Allow: /select/ | |
| 97 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 98 | + Allow: /subscribe/supplemental-terms | |
| 99 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 100 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 101 | + Allow: /tips | |
| 102 | + Allow: /tips/ | |
| 103 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 104 | + | |
| 69 | 105 | User-agent: Bytespider |
| 106 | + User-agent: ByteSpider Desktop | |
| 70 | 107 | Disallow: / |
| 71 | 108 | Allow: /select |
| 72 | 109 | Allow: /select/ |
| @@ -153,6 +190,18 @@ | ||
| 153 | 190 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 154 | 191 | |
| 155 | 192 | User-agent: Diffbot |
| 193 | + Disallow: / | |
| 194 | + Allow: /select | |
| 195 | + Allow: /select/ | |
| 196 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 197 | + Allow: /subscribe/supplemental-terms | |
| 198 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 199 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 200 | + Allow: /tips | |
| 201 | + Allow: /tips/ | |
| 202 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 203 | + | |
| 204 | + User-agent: ExaBot | |
| 156 | 205 | Disallow: / |
| 157 | 206 | Allow: /select |
| 158 | 207 | Allow: /select/ |
| @@ -321,15 +370,76 @@ | ||
| 321 | 370 | Allow: /tips/ |
| 322 | 371 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 323 | 372 | |
| 373 | + User-agent: Petalbot | |
| 374 | + Disallow: / | |
| 375 | + Allow: /select | |
| 376 | + Allow: /select/ | |
| 377 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 378 | + Allow: /subscribe/supplemental-terms | |
| 379 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 380 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 381 | + Allow: /tips | |
| 382 | + Allow: /tips/ | |
| 383 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 384 | + | |
| 385 | + User-agent: QuillBot | |
| 386 | + User-agent: quillbot.com | |
| 387 | + Disallow: / | |
| 388 | + Allow: /select | |
| 389 | + Allow: /select/ | |
| 390 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 391 | + Allow: /subscribe/supplemental-terms | |
| 392 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 393 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 394 | + Allow: /tips | |
| 395 | + Allow: /tips/ | |
| 396 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 397 | + | |
| 324 | 398 | User-agent: Scrapy |
| 325 | 399 | Disallow: / |
| 326 | 400 | Allow: /select |
| 327 | 401 | Allow: /select/ |
| 328 | 402 | |
| 403 | + User-agent: ShapBot | |
| 404 | + Disallow: / | |
| 405 | + Allow: /select | |
| 406 | + Allow: /select/ | |
| 407 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 408 | + Allow: /subscribe/supplemental-terms | |
| 409 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 410 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 411 | + Allow: /tips | |
| 412 | + Allow: /tips/ | |
| 413 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 414 | + | |
| 329 | 415 | User-agent: TurnitinBot |
| 330 | 416 | Disallow: / |
| 331 | 417 | Allow: /select |
| 332 | 418 | Allow: /select/ |
| 419 | + | |
| 420 | + User-agent: ViennaTinyBot | |
| 421 | + Disallow: / | |
| 422 | + Allow: /select | |
| 423 | + Allow: /select/ | |
| 424 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 425 | + Allow: /subscribe/supplemental-terms | |
| 426 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 427 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 428 | + Allow: /tips | |
| 429 | + Allow: /tips/ | |
| 430 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 431 | + | |
| 432 | + User-agent: YouBot | |
| 433 | + Disallow: / | |
| 434 | + Allow: /select | |
| 435 | + Allow: /select/ | |
| 436 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 437 | + Allow: /subscribe/supplemental-terms | |
| 438 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 439 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 440 | + Allow: /tips | |
| 441 | + Allow: /tips/ | |
| 442 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 333 | 443 | |
| 334 | 444 | # Sitemaps |
| 335 | 445 | Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-index |