NeuralCrawl

NBC News / robots.txt snapshot

← back to nbcnews.com · fetched 2026-10-22T02:51:55Z (16d ago) · HTTP 200 · 14591 bytes · sha256 9fc48b1225c428fc · raw

final URL: https://www.nbcnews.com/robots.txt

1User-agent: *
2Disallow: /search
3Disallow: /pages/search/
4Disallow: /pages/news-connect
5Disallow: /error404.aspx
6Disallow: /widget/
7Disallow: /*ns/local_news*
8Disallow: /akam/
9Disallow: /sigma.html
10Disallow: /sports/world-cup/fifa-world-cup-2026-bracket-challenge/group
11Disallow: /mynewsapi/
12
13Disallow: /*?*canonicalCard=
14User-agent: Twitterbot
15Allow: /*?*canonicalCard=
16
17# Disallow Bots (Allow /select only)
18User-agent: AhrefsBot
19Disallow: /
20Allow: /select
21Allow: /select/
22Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
23Allow: /subscribe/supplemental-terms
24Allow: /information/nbc-news-info/contact-us-n1232521
25Allow: /news/us-news/nbc-news-digital-editors-n893846
26Allow: /tips
27Allow: /tips/
28Allow: /information/nbc-news-info/closed-captioning-n1307063
29
30User-agent: Amazonbot
31User-agent: AmazonBot
32Disallow: /
33Allow: /select
34Allow: /select/
35Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
36Allow: /subscribe/supplemental-terms
37Allow: /information/nbc-news-info/contact-us-n1232521
38Allow: /news/us-news/nbc-news-digital-editors-n893846
39Allow: /tips
40Allow: /tips/
41Allow: /information/nbc-news-info/closed-captioning-n1307063
42
43User-agent: anthropic-ai
44Disallow: /
45Allow: /select
46Allow: /select/
47Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
48Allow: /subscribe/supplemental-terms
49Allow: /information/nbc-news-info/contact-us-n1232521
50Allow: /news/us-news/nbc-news-digital-editors-n893846
51Allow: /tips
52Allow: /tips/
53Allow: /information/nbc-news-info/closed-captioning-n1307063
54
55User-agent: Applebot-Extended
56Disallow: /
57Allow: /select
58Allow: /select/
59Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
60Allow: /subscribe/supplemental-terms
61Allow: /information/nbc-news-info/contact-us-n1232521
62Allow: /news/us-news/nbc-news-digital-editors-n893846
63Allow: /tips
64Allow: /tips/
65Allow: /information/nbc-news-info/closed-captioning-n1307063
66
67User-agent: Archive.org_bot
68Disallow: /
69Allow: /select
70Allow: /select/
71Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
72Allow: /subscribe/supplemental-terms
73Allow: /information/nbc-news-info/contact-us-n1232521
74Allow: /news/us-news/nbc-news-digital-editors-n893846
75Allow: /tips
76Allow: /tips/
77Allow: /information/nbc-news-info/closed-captioning-n1307063
78
79User-agent: AwarioRssBot
80User-agent: AwarioSmartBot
81Disallow: /
82Allow: /select
83Allow: /select/
84Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
85Allow: /subscribe/supplemental-terms
86Allow: /information/nbc-news-info/contact-us-n1232521
87Allow: /news/us-news/nbc-news-digital-editors-n893846
88Allow: /tips
89Allow: /tips/
90Allow: /information/nbc-news-info/closed-captioning-n1307063
91
92User-agent: Brightbot 1.0
93Disallow: /
94Allow: /select
95Allow: /select/
96Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
97Allow: /subscribe/supplemental-terms
98Allow: /information/nbc-news-info/contact-us-n1232521
99Allow: /news/us-news/nbc-news-digital-editors-n893846
100Allow: /tips
101Allow: /tips/
102Allow: /information/nbc-news-info/closed-captioning-n1307063
103
104User-agent: Bytespider
105User-agent: ByteSpider Desktop
106Disallow: /
107Allow: /select
108Allow: /select/
109Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
110Allow: /subscribe/supplemental-terms
111Allow: /information/nbc-news-info/contact-us-n1232521
112Allow: /news/us-news/nbc-news-digital-editors-n893846
113Allow: /tips
114Allow: /tips/
115Allow: /information/nbc-news-info/closed-captioning-n1307063
116
117User-agent: CCBot
118Disallow: /
119Allow: /select
120Allow: /select/
121Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
122Allow: /subscribe/supplemental-terms
123Allow: /information/nbc-news-info/contact-us-n1232521
124Allow: /news/us-news/nbc-news-digital-editors-n893846
125Allow: /tips
126Allow: /tips/
127Allow: /information/nbc-news-info/closed-captioning-n1307063
128
129User-agent: ChatGPT-User
130Disallow: /
131Allow: /select
132Allow: /select/
133Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
134Allow: /subscribe/supplemental-terms
135Allow: /information/nbc-news-info/contact-us-n1232521
136Allow: /news/us-news/nbc-news-digital-editors-n893846
137Allow: /tips
138Allow: /tips/
139Allow: /information/nbc-news-info/closed-captioning-n1307063
140
141User-agent: ClaudeBot
142Disallow: /
143Allow: /select
144Allow: /select/
145Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
146Allow: /subscribe/supplemental-terms
147Allow: /information/nbc-news-info/contact-us-n1232521
148Allow: /news/us-news/nbc-news-digital-editors-n893846
149Allow: /tips
150Allow: /tips/
151Allow: /information/nbc-news-info/closed-captioning-n1307063
152
153User-agent: Claude-Web
154User-agent: Claude-User
155User-agent: Claude-SearchBot
156Disallow: /
157Allow: /select
158Allow: /select/
159Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
160Allow: /subscribe/supplemental-terms
161Allow: /information/nbc-news-info/contact-us-n1232521
162Allow: /news/us-news/nbc-news-digital-editors-n893846
163Allow: /tips
164Allow: /tips/
165Allow: /information/nbc-news-info/closed-captioning-n1307063
166
167User-agent: cohere-ai
168Disallow: /
169Allow: /select
170Allow: /select/
171Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
172Allow: /subscribe/supplemental-terms
173Allow: /information/nbc-news-info/contact-us-n1232521
174Allow: /news/us-news/nbc-news-digital-editors-n893846
175Allow: /tips
176Allow: /tips/
177Allow: /information/nbc-news-info/closed-captioning-n1307063
178
179User-agent: DataForSeoBot
180Disallow: /
181Allow: /select
182Allow: /select/
183Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
184Allow: /subscribe/supplemental-terms
185Allow: /information/nbc-news-info/contact-us-n1232521
186Allow: /news/us-news/nbc-news-digital-editors-n893846
187Allow: /tips
188Allow: /tips/
189Allow: /information/nbc-news-info/closed-captioning-n1307063
190
191User-agent: Diffbot
192Disallow: /
193Allow: /select
194Allow: /select/
195Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
196Allow: /subscribe/supplemental-terms
197Allow: /information/nbc-news-info/contact-us-n1232521
198Allow: /news/us-news/nbc-news-digital-editors-n893846
199Allow: /tips
200Allow: /tips/
201Allow: /information/nbc-news-info/closed-captioning-n1307063
202
203User-agent: ExaBot
204User-agent: ExaSearchBot
205Disallow: /
206Allow: /select
207Allow: /select/
208Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
209Allow: /subscribe/supplemental-terms
210Allow: /information/nbc-news-info/contact-us-n1232521
211Allow: /news/us-news/nbc-news-digital-editors-n893846
212Allow: /tips
213Allow: /tips/
214Allow: /information/nbc-news-info/closed-captioning-n1307063
215
216User-agent: FacebookBot
217Disallow: /
218Allow: /select
219Allow: /select/
220
221User-agent: Google-Extended
222Disallow: /
223Allow: /select
224Allow: /select/
225Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
226Allow: /subscribe/supplemental-terms
227Allow: /information/nbc-news-info/contact-us-n1232521
228Allow: /news/us-news/nbc-news-digital-editors-n893846
229Allow: /tips
230Allow: /tips/
231Allow: /information/nbc-news-info/closed-captioning-n1307063
232
233User-agent: GPTBot
234Disallow: /
235Allow: /select
236Allow: /select/
237Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
238Allow: /subscribe/supplemental-terms
239Allow: /information/nbc-news-info/contact-us-n1232521
240Allow: /news/us-news/nbc-news-digital-editors-n893846
241Allow: /tips
242Allow: /tips/
243Allow: /information/nbc-news-info/closed-captioning-n1307063
244
245User-agent: ImageBot/1.0
246Disallow: /
247Allow: /select
248Allow: /select/
249Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
250Allow: /subscribe/supplemental-terms
251Allow: /information/nbc-news-info/contact-us-n1232521
252Allow: /news/us-news/nbc-news-digital-editors-n893846
253Allow: /tips
254Allow: /tips/
255Allow: /information/nbc-news-info/closed-captioning-n1307063
256
257User-agent: magpie-crawler
258Disallow: /
259Allow: /select
260Allow: /select/
261Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
262Allow: /subscribe/supplemental-terms
263Allow: /information/nbc-news-info/contact-us-n1232521
264Allow: /news/us-news/nbc-news-digital-editors-n893846
265Allow: /tips
266Allow: /tips/
267Allow: /information/nbc-news-info/closed-captioning-n1307063
268
269User-agent: Meta-ExternalAgent
270User-agent: meta-externalagent
271Disallow: /
272Allow: /select
273Allow: /select/
274Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
275Allow: /subscribe/supplemental-terms
276Allow: /information/nbc-news-info/contact-us-n1232521
277Allow: /news/us-news/nbc-news-digital-editors-n893846
278Allow: /tips
279Allow: /tips/
280Allow: /information/nbc-news-info/closed-captioning-n1307063
281
282User-agent: Meta-ExternalFetcher
283User-agent: meta-externalfetcher
284Disallow: /
285Allow: /select
286Allow: /select/
287Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
288Allow: /subscribe/supplemental-terms
289Allow: /information/nbc-news-info/contact-us-n1232521
290Allow: /news/us-news/nbc-news-digital-editors-n893846
291Allow: /tips
292Allow: /tips/
293Allow: /information/nbc-news-info/closed-captioning-n1307063
294
295User-agent: Meta-WebIndexer
296User-agent: meta-webindexer/1.1
297Disallow: /
298Allow: /shop
299Allow: /shop/
300Allow: /about/today/today-digital-masthead-rcna155416
301Allow: /about/today-rcna226078
302Allow: /information/join-today-live-plaza-t250868
303Allow: /start-today/app/supplemental-terms
304Allow: /information/closed-captioning-t215383
305Allow: /information/-us-send-today-your-questions-comments-t215381
306
307User-agent: MistralAI-User
308Disallow: /
309Allow: /select
310Allow: /select/
311Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
312Allow: /subscribe/supplemental-terms
313Allow: /information/nbc-news-info/contact-us-n1232521
314Allow: /news/us-news/nbc-news-digital-editors-n893846
315Allow: /tips
316Allow: /tips/
317Allow: /information/nbc-news-info/closed-captioning-n1307063
318
319User-agent: NagetBot
320Disallow: /
321Allow: /select
322Allow: /select/
323Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
324Allow: /subscribe/supplemental-terms
325Allow: /information/nbc-news-info/contact-us-n1232521
326Allow: /news/us-news/nbc-news-digital-editors-n893846
327Allow: /tips
328Allow: /tips/
329Allow: /information/nbc-news-info/closed-captioning-n1307063
330
331User-agent: NewsNow
332Disallow: /
333Allow: /select
334Allow: /select/
335
336User-agent: news-please
337Disallow: /
338Allow: /select
339Allow: /select/
340Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
341Allow: /subscribe/supplemental-terms
342Allow: /information/nbc-news-info/contact-us-n1232521
343Allow: /news/us-news/nbc-news-digital-editors-n893846
344Allow: /tips
345Allow: /tips/
346Allow: /information/nbc-news-info/closed-captioning-n1307063
347
348User-agent: OAI-SearchBot
349Disallow: /
350Allow: /select
351Allow: /select/
352Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
353Allow: /subscribe/supplemental-terms
354Allow: /information/nbc-news-info/contact-us-n1232521
355Allow: /news/us-news/nbc-news-digital-editors-n893846
356Allow: /tips
357Allow: /tips/
358Allow: /information/nbc-news-info/closed-captioning-n1307063
359
360User-agent: omgili
361User-agent: omgilibot
362Disallow: /
363Allow: /select
364Allow: /select/
365Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
366Allow: /subscribe/supplemental-terms
367Allow: /information/nbc-news-info/contact-us-n1232521
368Allow: /news/us-news/nbc-news-digital-editors-n893846
369Allow: /tips
370Allow: /tips/
371Allow: /information/nbc-news-info/closed-captioning-n1307063
372
373User-agent: peer39_crawler
374User-agent: peer39_crawler/1.0
375Disallow: /
376Allow: /select
377Allow: /select/
378Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
379Allow: /subscribe/supplemental-terms
380Allow: /information/nbc-news-info/contact-us-n1232521
381Allow: /news/us-news/nbc-news-digital-editors-n893846
382Allow: /tips
383Allow: /tips/
384Allow: /information/nbc-news-info/closed-captioning-n1307063
385
386User-agent: PerplexityBot
387User-agent: Perplexity-User
388Disallow: /
389Allow: /select
390Allow: /select/
391Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
392Allow: /subscribe/supplemental-terms
393Allow: /information/nbc-news-info/contact-us-n1232521
394Allow: /news/us-news/nbc-news-digital-editors-n893846
395Allow: /tips
396Allow: /tips/
397Allow: /information/nbc-news-info/closed-captioning-n1307063
398
399User-agent: Petalbot
400Disallow: /
401Allow: /select
402Allow: /select/
403Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
404Allow: /subscribe/supplemental-terms
405Allow: /information/nbc-news-info/contact-us-n1232521
406Allow: /news/us-news/nbc-news-digital-editors-n893846
407Allow: /tips
408Allow: /tips/
409Allow: /information/nbc-news-info/closed-captioning-n1307063
410
411User-agent: QuillBot
412User-agent: quillbot.com
413Disallow: /
414Allow: /select
415Allow: /select/
416Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
417Allow: /subscribe/supplemental-terms
418Allow: /information/nbc-news-info/contact-us-n1232521
419Allow: /news/us-news/nbc-news-digital-editors-n893846
420Allow: /tips
421Allow: /tips/
422Allow: /information/nbc-news-info/closed-captioning-n1307063
423
424User-agent: Scrapy
425Disallow: /
426Allow: /select
427Allow: /select/
428
429User-agent: ShapBot
430Disallow: /
431Allow: /select
432Allow: /select/
433Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
434Allow: /subscribe/supplemental-terms
435Allow: /information/nbc-news-info/contact-us-n1232521
436Allow: /news/us-news/nbc-news-digital-editors-n893846
437Allow: /tips
438Allow: /tips/
439Allow: /information/nbc-news-info/closed-captioning-n1307063
440
441User-agent: TaraGroup Intelligent Bot
442Disallow: /
443Allow: /select
444Allow: /select/
445Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
446Allow: /subscribe/supplemental-terms
447Allow: /information/nbc-news-info/contact-us-n1232521
448Allow: /news/us-news/nbc-news-digital-editors-n893846
449Allow: /tips
450Allow: /tips/
451Allow: /information/nbc-news-info/closed-captioning-n1307063
452
453User-agent: TurnitinBot
454Disallow: /
455Allow: /select
456Allow: /select/
457
458User-agent: ViennaTinyBot
459Disallow: /
460Allow: /select
461Allow: /select/
462Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
463Allow: /subscribe/supplemental-terms
464Allow: /information/nbc-news-info/contact-us-n1232521
465Allow: /news/us-news/nbc-news-digital-editors-n893846
466Allow: /tips
467Allow: /tips/
468Allow: /information/nbc-news-info/closed-captioning-n1307063
469
470User-agent: YouBot
471Disallow: /
472Allow: /select
473Allow: /select/
474Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
475Allow: /subscribe/supplemental-terms
476Allow: /information/nbc-news-info/contact-us-n1232521
477Allow: /news/us-news/nbc-news-digital-editors-n893846
478Allow: /tips
479Allow: /tips/
480Allow: /information/nbc-news-info/closed-captioning-n1307063
481
482# Sitemaps
483Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-index
484Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-news
485Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-curations
486Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-select.xml
487Sitemap: https://www.nbcnews.com/politics/election-results/sitemap.xml