NBC News / robots.txt
← back to nbcnews.com · change detected 18d ago (2026-10-20T02:14:13Z) · modified +25 −12
What changed
- Blocked user-agent ‘ExaSearchBot’ entirely
- Blocked user-agent ‘ImageBot/1.0’ entirely
- Blocked user-agent ‘TaraGroup Intelligent Bot’ entirely
- Removed Disallow: /bentoapi/ for *
Diff old (2026-07-27T21:55:10Z) → new (2026-10-20T02:14:13Z)
| @@ -5,7 +5,6 @@ | ||
| 5 | 5 | Disallow: /error404.aspx |
| 6 | 6 | Disallow: /widget/ |
| 7 | 7 | Disallow: /*ns/local_news* |
| 8 | - Disallow: /bentoapi/ | |
| 9 | 8 | Disallow: /akam/ |
| 10 | 9 | Disallow: /sigma.html |
| 11 | 10 | Disallow: /sports/world-cup/fifa-world-cup-2026-bracket-challenge/group |
| @@ -202,6 +201,7 @@ | ||
| 202 | 201 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 203 | 202 | |
| 204 | 203 | User-agent: ExaBot |
| 204 | + User-agent: ExaSearchBot | |
| 205 | 205 | Disallow: / |
| 206 | 206 | Allow: /select |
| 207 | 207 | Allow: /select/ |
| @@ -231,6 +231,18 @@ | ||
| 231 | 231 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 232 | 232 | |
| 233 | 233 | User-agent: GPTBot |
| 234 | + Disallow: / | |
| 235 | + Allow: /select | |
| 236 | + Allow: /select/ | |
| 237 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 238 | + Allow: /subscribe/supplemental-terms | |
| 239 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 240 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 241 | + Allow: /tips | |
| 242 | + Allow: /tips/ | |
| 243 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 244 | + | |
| 245 | + User-agent: ImageBot/1.0 | |
| 234 | 246 | Disallow: / |
| 235 | 247 | Allow: /select |
| 236 | 248 | Allow: /select/ |
| @@ -321,17 +333,6 @@ | ||
| 321 | 333 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 322 | 334 | |
| 323 | 335 | User-agent: omgili |
| 324 | - Disallow: / | |
| 325 | - Allow: /select | |
| 326 | - Allow: /select/ | |
| 327 | - Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 328 | - Allow: /subscribe/supplemental-terms | |
| 329 | - Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 330 | - Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 331 | - Allow: /tips | |
| 332 | - Allow: /tips/ | |
| 333 | - Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 334 | - | |
| 335 | 336 | User-agent: omgilibot |
| 336 | 337 | Disallow: / |
| 337 | 338 | Allow: /select |
| @@ -412,6 +413,18 @@ | ||
| 412 | 413 | Allow: /tips/ |
| 413 | 414 | Allow: /information/nbc-news-info/closed-captioning-n1307063 |
| 414 | 415 | |
| 416 | + User-agent: TaraGroup Intelligent Bot | |
| 417 | + Disallow: / | |
| 418 | + Allow: /select | |
| 419 | + Allow: /select/ | |
| 420 | + Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178 | |
| 421 | + Allow: /subscribe/supplemental-terms | |
| 422 | + Allow: /information/nbc-news-info/contact-us-n1232521 | |
| 423 | + Allow: /news/us-news/nbc-news-digital-editors-n893846 | |
| 424 | + Allow: /tips | |
| 425 | + Allow: /tips/ | |
| 426 | + Allow: /information/nbc-news-info/closed-captioning-n1307063 | |
| 427 | + | |
| 415 | 428 | User-agent: TurnitinBot |
| 416 | 429 | Disallow: / |
| 417 | 430 | Allow: /select |