NeuralCrawl

USA Today / robots.txt snapshot

← back to usatoday.com · fetched 2026-06-20T01:10:30Z (1mo ago) · HTTP 200 · 13310 bytes · sha256 3000fadc7d64bc3d · raw

final URL: https://www.usatoday.com/robots.txt

1# robots.txt file for https://www.usatoday.com/
2
3User-agent: anthropic-ai
4Disallow: /
5
6User-agent: AmazonBot
7Disallow: /
8
9User-agent: Claude
10Disallow: /
11
12User-agent: Claude-Web
13Disallow: /
14
15User-agent: claudeweb
16Disallow: /
17
18User-agent: Amzn-SearchBot
19Disallow: /
20
21User-agent: Amzn-User
22Disallow: /
23
24User-agent: AndiBot
25Disallow: /
26
27User-agent: cohere-ai
28Disallow: /
29
30User-agent: ClaudeBot
31Disallow: /
32
33User-agent: ChatGPT-User
34Disallow: /
35Allow: /contributor-content/
36Allow: /story/special/contributor-content/
37Allow: /shopping/
38
39User-agent: Googlebot-News
40Disallow: /story/sponsor-story/
41Disallow: /picture-gallery/sponsor-story/
42Disallow: /videos/sponsor-story/
43Disallow: /longform/sponsor-story/
44Disallow: /pages/interactives/sponsor-story/
45Disallow: /interactives/sponsor-story/
46Disallow: /videos/embed/
47
48User-agent: GPTBot
49Disallow: /
50Allow: /contributor-content/
51Allow: /story/special/contributor-content/
52Allow: /shopping/
53
54User-agent: CCBot
55Disallow: /
56
57User-agent: Google-Extended
58Disallow: /
59Allow: /contributor-content/
60Allow: /story/special/contributor-content/
61Allow: /shopping/
62
63User-agent: DataForSeoBot
64Disallow: /
65
66User-agent: Diffbot
67Disallow: /
68
69User-agent: Diffbot-user
70Disallow: /
71
72User-agent: Kagibot
73Disallow: /
74
75User-agent: FacebookBot
76Disallow: /
77
78User-agent: magpie-crawler
79Disallow: /
80
81User-agent: NewsNow
82Disallow: /
83
84User-agent: news-please
85Disallow: /
86
87User-agent: omgili
88Disallow: /
89
90User-agent: omgilibot
91Disallow: /
92
93User-agent: peer39_crawler
94Disallow: /
95
96User-agent: peer39_crawler/1.0
97Disallow: /
98
99User-agent: PerplexityBot
100Disallow: /
101Allow: /contributor-content/
102Allow: /story/special/contributor-content/
103Allow: /shopping/
104
105User-agent: Scrapy
106Disallow: /
107
108User-agent: TurnitinBot
109Disallow: /
110
111User-agent: Applebot-Extended
112Disallow: /
113
114User-agent: Meta-ExternalAgent
115Disallow: /
116
117User-agent: meta-externalagent
118Disallow: /
119
120User-agent: OAI-SearchBot
121Disallow: /
122Allow: /contributor-content/
123Allow: /story/special/contributor-content/
124Allow: /shopping/
125
126User-agent: Bytespider
127Disallow: /
128
129User-agent: Claude-User
130Disallow: /
131
132User-agent: Claude-SearchBot
133Disallow: /
134
135User-agent: AI2Bot
136Disallow: /
137
138User-agent: AhrefsBot
139Disallow: /
140
141User-agent: Ai2Bot-Dolma
142Disallow: /
143
144User-agent: Applebot
145Disallow: /
146
147User-agent: Arquivo-web-crawler
148Disallow: /
149
150User-agent: AspiegelBot
151Disallow: /
152
153User-agent: AwarioRssBot
154Disallow: /
155
156User-agent: AwarioSmartBot
157Disallow: /
158
159User-agent: Barkrowler
160Disallow: /
161
162User-agent: coccocbot-web
163Disallow: /
164
165User-agent: cohere-training-data-crawler
166Disallow: /
167
168User-agent: crawler4j
169Disallow: /
170
171User-agent: Crawlspace
172Disallow: /
173
174User-agent: dotbot
175Disallow: /
176
177User-agent: DuckAssistBot
178Disallow: /
179
180User-agent: FriendlyCrawler
181Disallow: /
182
183User-agent: Google-CloudVertexBot
184Disallow: /
185
186User-agent: https://hada.news
187Disallow: /
188
189User-agent: https://www.imediaethics.org
190Disallow: /
191
192User-agent: iaskspider/2.0
193Disallow: /
194
195User-agent: ICC-Crawler
196Disallow: /
197
198User-agent: icc-crawler
199Disallow: /
200
201User-agent: ICC Crawler
202Disallow: /
203
204User-agent: ImagesiftBot
205Disallow: /
206
207User-agent: img2dataset
208Disallow: /
209
210User-agent: ISSCyberRiskCrawler
211Disallow: /
212
213User-agent: JenkersBot
214Disallow: /
215
216User-agent: Kangaroo Bot
217Disallow: /
218
219User-agent: LivelapBot
220Disallow: /
221
222User-agent: MauiBot
223Disallow: /
224
225User-agent: Meltwater
226Disallow: /
227
228User-agent: Meta-ExternalFetcher
229Disallow: /
230
231User-agent: MJ12bot
232Disallow: /
233
234User-agent: Mojeek
235Disallow: /
236
237User-agent: MojeekBot
238Disallow: /
239
240User-agent: MoodleBot
241Disallow: /
242
243User-agent: NeticleBot
244Disallow: /
245
246User-agent: Netvibes
247Disallow: /
248
249User-agent: PanguBot
250Disallow: /
251
252User-agent: PetalBot
253Disallow: /
254
255User-agent: SeekportBot
256Disallow: /
257
258User-agent: Seekr
259Disallow: /
260
261User-agent: SemrushBot
262Disallow: /
263
264User-agent: SeznamBot
265Disallow: /
266
267User-agent: Sidetrade Indexer Bot
268Disallow: /
269
270User-agent: Timpibot
271Disallow: /
272
273User-agent: Turnitin
274Disallow: /
275
276User-agent: VelenPublicWebCrawler
277Disallow: /
278
279User-agent: Webzio-Extended
280Disallow: /
281
282User-agent: yacy
283Disallow: /
284
285User-agent: yacybot
286Disallow: /
287
288User-agent: Yandex
289Disallow: /
290
291User-agent: YandexAdditional
292Disallow: /
293
294User-agent: YandexAdditionalBot
295Disallow: /
296
297User-agent: YandexBot
298Disallow: /
299
300User-agent: YandexImages
301Disallow: /
302
303User-agent: YandexNews
304Disallow: /
305
306User-agent: Baiduspider
307Disallow: /
308
309User-agent: Baiduspider-image
310Disallow: /
311
312User-agent: BaiduSpider-render
313Disallow: /
314
315User-agent: Baiduspider-video
316Disallow: /
317
318User-agent: Sogou Web Spider
319Disallow: /
320
321User-agent: yisouspider
322Disallow: /
323
324User-agent: YouBot
325Disallow: /
326
327User-agent: ia_archiver
328Disallow: /
329
330User-agent: archive.org_bot
331Disallow: /
332
333User-agent: ia_archiver-web.archive.org
334Disallow: /
335
336User-agent: aiHitBot
337Disallow: /
338
339User-agent: Amazon Kendra
340Disallow: /
341
342User-agent: Big Sur AI
343Disallow: /
344
345User-agent: Brandwatch
346Disallow: /
347
348User-agent: Bravest
349Disallow: /
350
351User-agent: ChatGPT Operator
352Disallow: /
353Allow: /contributor-content/
354Allow: /story/special/contributor-content/
355Allow: /shopping/
356
357User-agent: ChatGPT-PageFetcher
358Disallow: /
359
360User-agent: OpenAI-CheckBot
361Disallow: /
362
363User-agent: OpenAI-Explore
364Disallow: /
365
366User-agent: OpenAI Feedback
367Disallow: /
368
369User-agent: OpenAI-SnippetBot
370Disallow: /
371
372User-agent: Cotoyogi
373Disallow: /
374
375User-agent: DigitalOceanGenAICrawler
376Disallow: /
377
378User-agent: Echobot Bot
379Disallow: /
380
381User-agent: EchoboxBot
382Disallow: /
383
384User-agent: Factset_spyderbot
385Disallow: /
386
387User-agent: Grok
388Disallow: /
389
390User-agent: iaskspider
391Disallow: /
392
393User-agent: imgproxy
394Disallow: /
395
396User-agent: LINER Bot
397Disallow: /
398
399User-agent: MistralAI-User
400Disallow: /
401
402User-agent: netEstate Imprint Crawler
403Disallow: /
404
405User-agent: NovaAct
406Disallow: /
407
408User-agent: QualifiedBot
409Disallow: /
410
411User-agent: SBIntuitionsBot
412Disallow: /
413
414User-agent: sbintuitionsbot
415Disallow: /
416
417User-agent: SemrushBot-OCOB
418Disallow: /
419
420User-agent: SemrushBotSwa
421Disallow: /
422
423User-agent: Sidetrade
424Disallow: /
425
426User-agent: TikTokSpider
427Disallow: /
428
429User-agent: WARDBot
430Disallow: /
431
432User-agent: TaraGroup Intelligent Bot V1
433Disallow: /
434
435User-agent: TaraGroup Intelligent Bot
436Disallow: /
437
438User-agent: DeepSeekBot
439Disallow: /
440
441User-agent: ChatGLM-Spider
442Disallow: /
443
444User-agent: GoogleOther
445Disallow: /
446
447User-agent: Google AI
448Disallow: /
449
450User-agent: Google-Agent
451Disallow: /
452
453User-agent: shapbot
454Disallow: /
455
456User-agent: FirecrawlAgent
457Disallow: /
458
459User-agent: Crawl4AI
460Disallow: /
461
462User-agent: imageSpider
463Disallow: /
464
465User-agent: laion-huggingface-processor
466Disallow: /
467
468User-agent: LCC
469Disallow: /
470
471User-agent: Spider
472Disallow: /
473
474User-agent: Datenbank Crawler
475Disallow: /
476
477User-agent: Cloudflare-AutoRAG
478Disallow: /
479
480User-agent: CloudflareBrowserRenderingCrawler
481Disallow: /
482
483User-agent: meta-webindexer
484Disallow: /
485
486User-agent: Bravebot
487Disallow: /
488
489User-agent: AddSearchBot
490Disallow: /
491
492User-agent: Anomura
493Disallow: /
494
495User-agent: atlassian-bot
496Disallow: /
497
498User-agent: Channel3Bot
499Disallow: /
500
501User-agent: LinkupBot
502Disallow: /
503
504User-agent: ZanistaBot
505Disallow: /
506
507User-agent: Gemini-Deep-Research
508Disallow: /
509
510User-agent: Google-NotebookLM
511Disallow: /
512
513User-agent: Perplexity-User
514Disallow: /
515
516User-agent: Perplexity Search Bot
517Disallow: /
518
519User-agent: Poetbot
520Disallow: /
521
522User-agent: Poseidon Research Crawler
523Disallow: /
524
525User-agent: ProRataInc
526Disallow: /
527
528User-agent: QuillBot
529Disallow: /
530
531User-agent: Tiingo News Agent
532Disallow: /
533
534User-agent: iboubot
535Disallow: /
536
537User-agent: TavilyBot
538Disallow: /
539
540User-agent: NeevaBot
541Disallow: /
542
543User-agent: PhindBot
544Disallow: /
545
546User-agent: AI2Bot-DeepResearchEval
547Disallow: /
548
549User-agent: KlaviyoAIBot
550Disallow: /
551
552User-agent: LinerBot
553Disallow: /
554
555User-agent: Poggio-Citations
556Disallow: /
557
558User-agent: Devin
559Disallow: /
560
561User-agent: bigsur.ai
562Disallow: /
563
564User-agent: ChatGPT Agent
565Disallow: /
566
567User-agent: GoogleAgent-Mariner
568Disallow: /
569
570User-agent: AmazonBuyForMe
571Disallow: /
572
573User-agent: Manus-User
574Disallow: /
575
576User-agent: manus-user
577Disallow: /
578
579User-agent: TwinAgent
580Disallow: /
581
582User-agent: AudigentAdBot
583Disallow: /
584
585User-agent: AudigentBot
586Disallow: /
587
588User-agent: JinaAI
589Disallow: /
590
591User-agent: Jina-AI
592Disallow: /
593
594User-agent: JinaBot
595Disallow: /
596
597User-agent: Jina-Reader
598Disallow: /
599
600User-agent: BrightData
601Disallow: /
602
603User-agent: Bright Data
604Disallow: /
605
606User-agent: BrightBot
607Disallow: /
608
609User-agent: SerpApi
610Disallow: /
611
612User-agent: SerpApiBot
613Disallow: /
614
615User-agent: Apify
616Disallow: /
617
618User-agent: ApifyBot
619Disallow: /
620
621User-agent: Apify-HTTP-Client
622Disallow: /
623
624User-agent: ScrapeGraphAI
625Disallow: /
626
627User-agent: scrapegraph
628Disallow: /
629
630User-agent: Scrapeless
631Disallow: /
632
633User-agent: ScrapelessBot
634Disallow: /
635
636User-agent: ZenRows
637Disallow: /
638
639User-agent: ZenRowsBot
640Disallow: /
641
642User-agent: PromptCloud
643Disallow: /
644
645User-agent: PromptCloudBot
646Disallow: /
647
648User-agent: Nimbleway
649Disallow: /
650
651User-agent: NimbleCrawler
652Disallow: /
653
654User-agent: NimblewayBot
655Disallow: /
656
657User-agent: Oxylabs
658Disallow: /
659
660User-agent: OxylabsBot
661Disallow: /
662
663User-agent: Zyte
664Disallow: /
665
666User-agent: ZyteBot
667Disallow: /
668
669User-agent: Scrapinghub
670Disallow: /
671
672User-agent: ScraperAPI
673Disallow: /
674
675User-agent: Scraper-API
676Disallow: /
677
678User-agent: ScrapingBee
679Disallow: /
680
681User-agent: ScrapingBeeBot
682Disallow: /
683
684User-agent: Scrapingdog
685Disallow: /
686
687User-agent: ScrapingdogBot
688Disallow: /
689
690User-agent: Grepsr
691Disallow: /
692
693User-agent: GrepsrBot
694Disallow: /
695
696User-agent: GrepsrCrawler
697Disallow: /
698
699User-agent: SpiderBot
700Disallow: /
701
702User-agent: spider-cloud
703Disallow: /
704
705User-agent: SpiderCloud
706Disallow: /
707
708User-agent: Hyperbrowser
709Disallow: /
710
711User-agent: HyperbrowserBot
712Disallow: /
713
714User-agent: HyperAgent
715Disallow: /
716
717User-agent: Scrapestack
718Disallow: /
719
720User-agent: ScrapestackBot
721Disallow: /
722
723User-agent: WebScraping.AI
724Disallow: /
725
726User-agent: WebScrapingAI
727Disallow: /
728
729User-agent: webscraping-ai
730Disallow: /
731
732User-agent: ForageAI
733Disallow: /
734
735User-agent: Forage-AI
736Disallow: /
737
738User-agent: ForageBot
739Disallow: /
740
741User-agent: DumplingAI
742Disallow: /
743
744User-agent: Dumpling-AI
745Disallow: /
746
747User-agent: DumplingBot
748Disallow: /
749
750User-agent: Import.io
751Disallow: /
752
753User-agent: ImportIO
754Disallow: /
755
756User-agent: importio
757Disallow: /
758
759User-agent: BrowseAI
760Disallow: /
761
762User-agent: Browse-AI
763Disallow: /
764
765User-agent: BrowseAIBot
766Disallow: /
767
768User-agent: Decodo
769Disallow: /
770
771User-agent: DecodoBot
772Disallow: /
773
774User-agent: Smartproxy
775Disallow: /
776
777User-agent: Scrapfly
778Disallow: /
779
780User-agent: ScrapflyBot
781Disallow: /
782
783User-agent: ScrapflyClient
784Disallow: /
785
786User-agent: Octoparse
787Disallow: /
788
789User-agent: OctoparseBot
790Disallow: /
791
792User-agent: TrajectData
793Disallow: /
794
795User-agent: Traject-Data
796Disallow: /
797
798User-agent: SerpWow
799Disallow: /
800
801User-agent: ScaleSERP
802Disallow: /
803
804User-agent: Rainforest-API
805Disallow: /
806
807User-agent: BlueCart-API
808Disallow: /
809
810User-agent: skrape.ai
811Disallow: /
812
813User-agent: SkrapeAI
814Disallow: /
815
816User-agent: SkrapeBot
817Disallow: /
818
819User-agent: Froxy
820Disallow: /
821
822User-agent: FroxyBot
823Disallow: /
824
825User-agent: MrScraper
826Disallow: /
827
828User-agent: Mr-Scraper
829Disallow: /
830
831User-agent: MrScraperBot
832Disallow: /
833
834User-agent: CrawlNow
835Disallow: /
836
837User-agent: CrawlNowBot
838Disallow: /
839
840User-agent: Soax
841Disallow: /
842
843User-agent: SoaxBot
844Disallow: /
845
846User-agent: SOAX
847Disallow: /
848
849User-agent: Kadoa
850Disallow: /
851
852User-agent: Kadoabot
853Disallow: /
854
855User-agent: KadoaBot
856Disallow: /
857
858User-agent: ScrapeHero
859Disallow: /
860
861User-agent: Scrape-Hero
862Disallow: /
863
864User-agent: ScrapeHeroBot
865Disallow: /
866
867User-agent: Exa
868Disallow: /
869
870User-agent: ExaBot
871Disallow: /
872
873User-agent: Exa-AI
874Disallow: /
875
876User-agent: ExaCrawler
877Disallow: /
878
879User-agent: Exabot
880Disallow: /
881
882User-agent: *
883Disallow: /errors
884Disallow: /interactive/
885Disallow: /userauth/
886Disallow: /ugc/
887Disallow: /feeds/
888Disallow: /services/
889Disallow: /facebook/
890Disallow: /version-info/
891Disallow: /longform/draft/
892Disallow: /story/draft/
893Disallow: /topic/*/smart/
894Disallow: /search
895Disallow: /module-showcase/
896Disallow: /newsletter/
897Disallow: /blended-newsletter/
898Disallow: /story/nletter/
899Disallow: /sports/services/photos/
900Disallow: /optimus
901Disallow: /ux-train
902Disallow: /story/advisory/
903Disallow: /.cam-tangent/
904Disallow: /pbd/
905Disallow: /gciaf/
906Disallow: /gcdn/gciaf/
907Disallow: /dcc/
908Disallow: /gcdn/dcc/
909Disallow: /dc/
910Disallow: /gcdn/dc/
911Disallow: /dcjs/
912Disallow: /gcdn/dcjs/
913Disallow: /content-queries/
914Disallow: /zxc/
915Disallow: /money/homefront/
916Disallow: /exp-cruise
917Disallow: /exp-las-vegas2
918Disallow: /exp-faw
919Disallow: /exp-caribbean
920Disallow: /exp-beach
921Disallow: /exp-cruise2
922Disallow: /yourtake
923Disallow: /story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/
924Disallow: /story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/
925Disallow: /story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/
926Disallow: /story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/
927Disallow: /money/lookup/stocks/
928Disallow: /money/blueprint/archive/
929Disallow: /money/blueprint/transfer?pname=
930
931User-agent: AmazonAdBot
932Allow: /
933
934Sitemap: https://www.usatoday.com/news-sitemap.xml
935Sitemap: https://www.usatoday.com/web-sitemap-index.xml
936Sitemap: https://www.usatoday.com/video-sitemap-index.xml
937Sitemap: https://www.usatoday.com/money/blueprint/sitemap.xml
938Sitemap: https://www.usatoday.com/money/blueprint/sitemap-news.xml
939Sitemap: https://www.usatoday.com/online-betting/sitemap.xml
940Sitemap: https://www.usatoday.com/online-betting/news-sitemap.xml