NeuralCrawl

NBC News / robots.txt snapshot

← back to nbcnews.com · fetched 2026-10-22T21:05:52Z (15d ago) · HTTP 200 · 14603 bytes · sha256 3c68189adff5155e · raw

final URL: https://www.nbcnews.com/robots.txt

1User-agent: *
2Disallow: /search
3Disallow: /pages/search/
4Disallow: /pages/news-connect
5Disallow: /error404.aspx
6Disallow: /widget/
7Disallow: /*ns/local_news*
8Disallow: /akam/
9Disallow: /sigma.html
10Disallow: /sports/world-cup/fifa-world-cup-2026-bracket-challenge/group
11Disallow: /mynewsapi/
12
13Disallow: /*?*canonicalCard=
14User-agent: Twitterbot
15Allow: /*?*canonicalCard=
16
17# Disallow Bots (Allow /select only)
18User-agent: AhrefsBot
19Disallow: /
20Allow: /select
21Allow: /select/
22Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
23Allow: /subscribe/supplemental-terms
24Allow: /information/nbc-news-info/contact-us-n1232521
25Allow: /news/us-news/nbc-news-digital-editors-n893846
26Allow: /tips
27Allow: /tips/
28Allow: /information/nbc-news-info/closed-captioning-n1307063
29
30User-agent: Amazonbot
31User-agent: AmazonBot
32Disallow: /
33Allow: /select
34Allow: /select/
35Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
36Allow: /subscribe/supplemental-terms
37Allow: /information/nbc-news-info/contact-us-n1232521
38Allow: /news/us-news/nbc-news-digital-editors-n893846
39Allow: /tips
40Allow: /tips/
41Allow: /information/nbc-news-info/closed-captioning-n1307063
42
43User-agent: anthropic-ai
44Disallow: /
45Allow: /select
46Allow: /select/
47Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
48Allow: /subscribe/supplemental-terms
49Allow: /information/nbc-news-info/contact-us-n1232521
50Allow: /news/us-news/nbc-news-digital-editors-n893846
51Allow: /tips
52Allow: /tips/
53Allow: /information/nbc-news-info/closed-captioning-n1307063
54
55User-agent: Applebot-Extended
56Disallow: /
57Allow: /select
58Allow: /select/
59Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
60Allow: /subscribe/supplemental-terms
61Allow: /information/nbc-news-info/contact-us-n1232521
62Allow: /news/us-news/nbc-news-digital-editors-n893846
63Allow: /tips
64Allow: /tips/
65Allow: /information/nbc-news-info/closed-captioning-n1307063
66
67User-agent: Archive.org_bot
68Disallow: /
69Allow: /select
70Allow: /select/
71Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
72Allow: /subscribe/supplemental-terms
73Allow: /information/nbc-news-info/contact-us-n1232521
74Allow: /news/us-news/nbc-news-digital-editors-n893846
75Allow: /tips
76Allow: /tips/
77Allow: /information/nbc-news-info/closed-captioning-n1307063
78
79User-agent: AwarioRssBot
80User-agent: AwarioSmartBot
81Disallow: /
82Allow: /select
83Allow: /select/
84Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
85Allow: /subscribe/supplemental-terms
86Allow: /information/nbc-news-info/contact-us-n1232521
87Allow: /news/us-news/nbc-news-digital-editors-n893846
88Allow: /tips
89Allow: /tips/
90Allow: /information/nbc-news-info/closed-captioning-n1307063
91
92User-agent: Brightbot 1.0
93Disallow: /
94Allow: /select
95Allow: /select/
96Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
97Allow: /subscribe/supplemental-terms
98Allow: /information/nbc-news-info/contact-us-n1232521
99Allow: /news/us-news/nbc-news-digital-editors-n893846
100Allow: /tips
101Allow: /tips/
102Allow: /information/nbc-news-info/closed-captioning-n1307063
103
104User-agent: Bytespider
105User-agent: ByteSpider Desktop
106Disallow: /
107Allow: /select
108Allow: /select/
109Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
110Allow: /subscribe/supplemental-terms
111Allow: /information/nbc-news-info/contact-us-n1232521
112Allow: /news/us-news/nbc-news-digital-editors-n893846
113Allow: /tips
114Allow: /tips/
115Allow: /information/nbc-news-info/closed-captioning-n1307063
116
117User-agent: CCBot
118Disallow: /
119Allow: /select
120Allow: /select/
121Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
122Allow: /subscribe/supplemental-terms
123Allow: /information/nbc-news-info/contact-us-n1232521
124Allow: /news/us-news/nbc-news-digital-editors-n893846
125Allow: /tips
126Allow: /tips/
127Allow: /information/nbc-news-info/closed-captioning-n1307063
128
129User-agent: ChatGPT-User
130Disallow: /
131Allow: /select
132Allow: /select/
133Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
134Allow: /subscribe/supplemental-terms
135Allow: /information/nbc-news-info/contact-us-n1232521
136Allow: /news/us-news/nbc-news-digital-editors-n893846
137Allow: /tips
138Allow: /tips/
139Allow: /information/nbc-news-info/closed-captioning-n1307063
140
141User-agent: ClaudeBot
142Disallow: /
143Allow: /select
144Allow: /select/
145Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
146Allow: /subscribe/supplemental-terms
147Allow: /information/nbc-news-info/contact-us-n1232521
148Allow: /news/us-news/nbc-news-digital-editors-n893846
149Allow: /tips
150Allow: /tips/
151Allow: /information/nbc-news-info/closed-captioning-n1307063
152
153User-agent: Claude-Web
154User-agent: Claude-User
155User-agent: Claude-SearchBot
156Disallow: /
157Allow: /select
158Allow: /select/
159Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
160Allow: /subscribe/supplemental-terms
161Allow: /information/nbc-news-info/contact-us-n1232521
162Allow: /news/us-news/nbc-news-digital-editors-n893846
163Allow: /tips
164Allow: /tips/
165Allow: /information/nbc-news-info/closed-captioning-n1307063
166
167User-agent: cohere-ai
168Disallow: /
169Allow: /select
170Allow: /select/
171Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
172Allow: /subscribe/supplemental-terms
173Allow: /information/nbc-news-info/contact-us-n1232521
174Allow: /news/us-news/nbc-news-digital-editors-n893846
175Allow: /tips
176Allow: /tips/
177Allow: /information/nbc-news-info/closed-captioning-n1307063
178
179User-agent: DataForSeoBot
180Disallow: /
181Allow: /select
182Allow: /select/
183Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
184Allow: /subscribe/supplemental-terms
185Allow: /information/nbc-news-info/contact-us-n1232521
186Allow: /news/us-news/nbc-news-digital-editors-n893846
187Allow: /tips
188Allow: /tips/
189Allow: /information/nbc-news-info/closed-captioning-n1307063
190
191User-agent: Diffbot
192Disallow: /
193Allow: /select
194Allow: /select/
195Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
196Allow: /subscribe/supplemental-terms
197Allow: /information/nbc-news-info/contact-us-n1232521
198Allow: /news/us-news/nbc-news-digital-editors-n893846
199Allow: /tips
200Allow: /tips/
201Allow: /information/nbc-news-info/closed-captioning-n1307063
202
203User-agent: ExaBot
204User-agent: ExaSearchBot
205Disallow: /
206Allow: /select
207Allow: /select/
208Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
209Allow: /subscribe/supplemental-terms
210Allow: /information/nbc-news-info/contact-us-n1232521
211Allow: /news/us-news/nbc-news-digital-editors-n893846
212Allow: /tips
213Allow: /tips/
214Allow: /information/nbc-news-info/closed-captioning-n1307063
215
216User-agent: FacebookBot
217Disallow: /
218Allow: /select
219Allow: /select/
220
221User-agent: Google-Extended
222Disallow: /
223Allow: /select
224Allow: /select/
225Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
226Allow: /subscribe/supplemental-terms
227Allow: /information/nbc-news-info/contact-us-n1232521
228Allow: /news/us-news/nbc-news-digital-editors-n893846
229Allow: /tips
230Allow: /tips/
231Allow: /information/nbc-news-info/closed-captioning-n1307063
232
233User-agent: GPTBot
234Disallow: /
235Allow: /select
236Allow: /select/
237Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
238Allow: /subscribe/supplemental-terms
239Allow: /information/nbc-news-info/contact-us-n1232521
240Allow: /news/us-news/nbc-news-digital-editors-n893846
241Allow: /tips
242Allow: /tips/
243Allow: /information/nbc-news-info/closed-captioning-n1307063
244
245User-agent: ImageBot/1.0
246Disallow: /
247Allow: /select
248Allow: /select/
249Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
250Allow: /subscribe/supplemental-terms
251Allow: /information/nbc-news-info/contact-us-n1232521
252Allow: /news/us-news/nbc-news-digital-editors-n893846
253Allow: /tips
254Allow: /tips/
255Allow: /information/nbc-news-info/closed-captioning-n1307063
256
257User-agent: magpie-crawler
258Disallow: /
259Allow: /select
260Allow: /select/
261Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
262Allow: /subscribe/supplemental-terms
263Allow: /information/nbc-news-info/contact-us-n1232521
264Allow: /news/us-news/nbc-news-digital-editors-n893846
265Allow: /tips
266Allow: /tips/
267Allow: /information/nbc-news-info/closed-captioning-n1307063
268
269User-agent: Meta-ExternalAgent
270User-agent: meta-externalagent
271Disallow: /
272Allow: /select
273Allow: /select/
274Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
275Allow: /subscribe/supplemental-terms
276Allow: /information/nbc-news-info/contact-us-n1232521
277Allow: /news/us-news/nbc-news-digital-editors-n893846
278Allow: /tips
279Allow: /tips/
280Allow: /information/nbc-news-info/closed-captioning-n1307063
281
282User-agent: Meta-ExternalFetcher
283User-agent: meta-externalfetcher
284Disallow: /
285Allow: /select
286Allow: /select/
287Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
288Allow: /subscribe/supplemental-terms
289Allow: /information/nbc-news-info/contact-us-n1232521
290Allow: /news/us-news/nbc-news-digital-editors-n893846
291Allow: /tips
292Allow: /tips/
293Allow: /information/nbc-news-info/closed-captioning-n1307063
294
295User-agent: Meta-WebIndexer
296User-agent: meta-webindexer/1.1
297Disallow: /
298Allow: /select
299Allow: /select/
300Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
301Allow: /subscribe/supplemental-terms
302Allow: /information/nbc-news-info/contact-us-n1232521
303Allow: /news/us-news/nbc-news-digital-editors-n893846
304Allow: /tips
305Allow: /tips/
306Allow: /information/nbc-news-info/closed-captioning-n1307063
307
308User-agent: MistralAI-User
309Disallow: /
310Allow: /select
311Allow: /select/
312Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
313Allow: /subscribe/supplemental-terms
314Allow: /information/nbc-news-info/contact-us-n1232521
315Allow: /news/us-news/nbc-news-digital-editors-n893846
316Allow: /tips
317Allow: /tips/
318Allow: /information/nbc-news-info/closed-captioning-n1307063
319
320User-agent: NagetBot
321Disallow: /
322Allow: /select
323Allow: /select/
324Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
325Allow: /subscribe/supplemental-terms
326Allow: /information/nbc-news-info/contact-us-n1232521
327Allow: /news/us-news/nbc-news-digital-editors-n893846
328Allow: /tips
329Allow: /tips/
330Allow: /information/nbc-news-info/closed-captioning-n1307063
331
332User-agent: NewsNow
333Disallow: /
334Allow: /select
335Allow: /select/
336
337User-agent: news-please
338Disallow: /
339Allow: /select
340Allow: /select/
341Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
342Allow: /subscribe/supplemental-terms
343Allow: /information/nbc-news-info/contact-us-n1232521
344Allow: /news/us-news/nbc-news-digital-editors-n893846
345Allow: /tips
346Allow: /tips/
347Allow: /information/nbc-news-info/closed-captioning-n1307063
348
349User-agent: OAI-SearchBot
350Disallow: /
351Allow: /select
352Allow: /select/
353Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
354Allow: /subscribe/supplemental-terms
355Allow: /information/nbc-news-info/contact-us-n1232521
356Allow: /news/us-news/nbc-news-digital-editors-n893846
357Allow: /tips
358Allow: /tips/
359Allow: /information/nbc-news-info/closed-captioning-n1307063
360
361User-agent: omgili
362User-agent: omgilibot
363Disallow: /
364Allow: /select
365Allow: /select/
366Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
367Allow: /subscribe/supplemental-terms
368Allow: /information/nbc-news-info/contact-us-n1232521
369Allow: /news/us-news/nbc-news-digital-editors-n893846
370Allow: /tips
371Allow: /tips/
372Allow: /information/nbc-news-info/closed-captioning-n1307063
373
374User-agent: peer39_crawler
375User-agent: peer39_crawler/1.0
376Disallow: /
377Allow: /select
378Allow: /select/
379Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
380Allow: /subscribe/supplemental-terms
381Allow: /information/nbc-news-info/contact-us-n1232521
382Allow: /news/us-news/nbc-news-digital-editors-n893846
383Allow: /tips
384Allow: /tips/
385Allow: /information/nbc-news-info/closed-captioning-n1307063
386
387User-agent: PerplexityBot
388User-agent: Perplexity-User
389Disallow: /
390Allow: /select
391Allow: /select/
392Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
393Allow: /subscribe/supplemental-terms
394Allow: /information/nbc-news-info/contact-us-n1232521
395Allow: /news/us-news/nbc-news-digital-editors-n893846
396Allow: /tips
397Allow: /tips/
398Allow: /information/nbc-news-info/closed-captioning-n1307063
399
400User-agent: Petalbot
401Disallow: /
402Allow: /select
403Allow: /select/
404Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
405Allow: /subscribe/supplemental-terms
406Allow: /information/nbc-news-info/contact-us-n1232521
407Allow: /news/us-news/nbc-news-digital-editors-n893846
408Allow: /tips
409Allow: /tips/
410Allow: /information/nbc-news-info/closed-captioning-n1307063
411
412User-agent: QuillBot
413User-agent: quillbot.com
414Disallow: /
415Allow: /select
416Allow: /select/
417Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
418Allow: /subscribe/supplemental-terms
419Allow: /information/nbc-news-info/contact-us-n1232521
420Allow: /news/us-news/nbc-news-digital-editors-n893846
421Allow: /tips
422Allow: /tips/
423Allow: /information/nbc-news-info/closed-captioning-n1307063
424
425User-agent: Scrapy
426Disallow: /
427Allow: /select
428Allow: /select/
429
430User-agent: ShapBot
431Disallow: /
432Allow: /select
433Allow: /select/
434Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
435Allow: /subscribe/supplemental-terms
436Allow: /information/nbc-news-info/contact-us-n1232521
437Allow: /news/us-news/nbc-news-digital-editors-n893846
438Allow: /tips
439Allow: /tips/
440Allow: /information/nbc-news-info/closed-captioning-n1307063
441
442User-agent: TaraGroup Intelligent Bot
443Disallow: /
444Allow: /select
445Allow: /select/
446Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
447Allow: /subscribe/supplemental-terms
448Allow: /information/nbc-news-info/contact-us-n1232521
449Allow: /news/us-news/nbc-news-digital-editors-n893846
450Allow: /tips
451Allow: /tips/
452Allow: /information/nbc-news-info/closed-captioning-n1307063
453
454User-agent: TurnitinBot
455Disallow: /
456Allow: /select
457Allow: /select/
458
459User-agent: ViennaTinyBot
460Disallow: /
461Allow: /select
462Allow: /select/
463Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
464Allow: /subscribe/supplemental-terms
465Allow: /information/nbc-news-info/contact-us-n1232521
466Allow: /news/us-news/nbc-news-digital-editors-n893846
467Allow: /tips
468Allow: /tips/
469Allow: /information/nbc-news-info/closed-captioning-n1307063
470
471User-agent: YouBot
472Disallow: /
473Allow: /select
474Allow: /select/
475Allow: /information/nbc-news-info/about-nbc-news-digital-n1232178
476Allow: /subscribe/supplemental-terms
477Allow: /information/nbc-news-info/contact-us-n1232521
478Allow: /news/us-news/nbc-news-digital-editors-n893846
479Allow: /tips
480Allow: /tips/
481Allow: /information/nbc-news-info/closed-captioning-n1307063
482
483# Sitemaps
484Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-index
485Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-news
486Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-curations
487Sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-select.xml
488Sitemap: https://www.nbcnews.com/politics/election-results/sitemap.xml