NeuralCrawl

USA Today / robots.txt snapshot

← back to usatoday.com · fetched 2026-09-29T00:37:36Z (1mo ago) · HTTP 200 · 16350 bytes · sha256 7ac5cfceb70ab70e · raw

final URL: https://www.usatoday.com/robots.txt

1# robots.txt file for https://www.usatoday.com/
2
3User-agent: anthropic-ai
4Disallow: /
5
6User-agent: AmazonBot
7Disallow: /
8
9User-agent: Claude
10Disallow: /
11
12User-agent: Claude-Web
13Disallow: /
14
15User-agent: claudeweb
16Disallow: /
17
18User-agent: Amzn-SearchBot
19Disallow: /
20
21User-agent: Amzn-User
22Disallow: /
23
24User-agent: AndiBot
25Disallow: /
26
27User-agent: cohere-ai
28Disallow: /
29
30User-agent: ClaudeBot
31Disallow: /
32
33User-agent: ChatGPT-User
34Disallow: /
35Allow: /branded-story/
36Allow: /contributor-content/
37Allow: /story/branded-story/
38Allow: /story/special/contributor-content/
39Allow: /money/money-management/
40Allow: /shopping/
41Allow: /story/shopping/
42Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/
43Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/
44Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/
45
46User-agent: Googlebot-News
47Disallow: /story/sponsor-story/
48Disallow: /picture-gallery/sponsor-story/
49Disallow: /videos/sponsor-story/
50Disallow: /longform/sponsor-story/
51Disallow: /pages/interactives/sponsor-story/
52Disallow: /interactives/sponsor-story/
53Disallow: /videos/embed/
54
55User-agent: GPTBot
56Disallow: /
57Allow: /branded-story/
58Allow: /contributor-content/
59Allow: /story/branded-story/
60Allow: /story/special/contributor-content/
61Allow: /money/money-management/
62Allow: /shopping/
63Allow: /story/shopping/
64Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/
65Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/
66Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/
67
68User-agent: CCBot
69Disallow: /
70
71User-agent: Google-Extended
72Disallow: /
73Allow: /branded-story/
74Allow: /contributor-content/
75Allow: /story/branded-story/
76Allow: /story/special/contributor-content/
77Allow: /money/money-management/
78Allow: /shopping/
79Allow: /story/shopping/
80Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/
81Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/
82Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/
83
84User-agent: DataForSeoBot
85Disallow: /
86
87User-agent: Diffbot
88Disallow: /
89
90User-agent: Diffbot-user
91Disallow: /
92
93User-agent: Kagibot
94Disallow: /
95
96User-agent: FacebookBot
97Disallow: /
98
99User-agent: magpie-crawler
100Disallow: /
101
102User-agent: NewsNow
103Disallow: /
104
105User-agent: news-please
106Disallow: /
107
108User-agent: omgili
109Disallow: /
110
111User-agent: omgilibot
112Disallow: /
113
114User-agent: peer39_crawler
115Disallow: /
116
117User-agent: peer39_crawler/1.0
118Disallow: /
119
120User-agent: PerplexityBot
121Disallow: /
122Allow: /branded-story/
123Allow: /contributor-content/
124Allow: /story/branded-story/
125Allow: /story/special/contributor-content/
126Allow: /money/money-management/
127Allow: /shopping/
128Allow: /story/shopping/
129Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/
130Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/
131Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/
132
133User-agent: Scrapy
134Disallow: /
135
136User-agent: TurnitinBot
137Disallow: /
138
139User-agent: Applebot-Extended
140Disallow: /
141
142User-agent: Meta-ExternalAgent
143Disallow: /
144
145User-agent: meta-externalagent
146Disallow: /
147
148User-agent: OAI-SearchBot
149Disallow: /
150Allow: /branded-story/
151Allow: /contributor-content/
152Allow: /story/branded-story/
153Allow: /story/special/contributor-content/
154Allow: /money/money-management/
155Allow: /shopping/
156Allow: /story/shopping/
157Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/
158Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/
159Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/
160
161User-agent: Bytespider
162Disallow: /
163
164User-agent: Claude-User
165Disallow: /
166
167User-agent: Claude-SearchBot
168Disallow: /
169
170User-agent: AI2Bot
171Disallow: /
172
173User-agent: AhrefsBot
174Disallow: /
175
176User-agent: Ai2Bot-Dolma
177Disallow: /
178
179User-agent: Applebot
180Disallow: /
181
182User-agent: Arquivo-web-crawler
183Disallow: /
184
185User-agent: AspiegelBot
186Disallow: /
187
188User-agent: AwarioRssBot
189Disallow: /
190
191User-agent: AwarioSmartBot
192Disallow: /
193
194User-agent: Barkrowler
195Disallow: /
196
197User-agent: coccocbot-web
198Disallow: /
199
200User-agent: cohere-training-data-crawler
201Disallow: /
202
203User-agent: crawler4j
204Disallow: /
205
206User-agent: Crawlspace
207Disallow: /
208
209User-agent: dotbot
210Disallow: /
211
212User-agent: DuckAssistBot
213Disallow: /
214
215User-agent: FriendlyCrawler
216Disallow: /
217
218User-agent: Google-CloudVertexBot
219Disallow: /
220
221User-agent: https://hada.news
222Disallow: /
223
224User-agent: https://www.imediaethics.org
225Disallow: /
226
227User-agent: iaskspider/2.0
228Disallow: /
229
230User-agent: ICC-Crawler
231Disallow: /
232
233User-agent: icc-crawler
234Disallow: /
235
236User-agent: ICC Crawler
237Disallow: /
238
239User-agent: ImagesiftBot
240Disallow: /
241
242User-agent: img2dataset
243Disallow: /
244
245User-agent: ISSCyberRiskCrawler
246Disallow: /
247
248User-agent: JenkersBot
249Disallow: /
250
251User-agent: Kangaroo Bot
252Disallow: /
253
254User-agent: LivelapBot
255Disallow: /
256
257User-agent: MauiBot
258Disallow: /
259
260User-agent: Meltwater
261Disallow: /
262
263User-agent: Meta-ExternalFetcher
264Disallow: /
265
266User-agent: MJ12bot
267Disallow: /
268
269User-agent: Mojeek
270Disallow: /
271
272User-agent: MojeekBot
273Disallow: /
274
275User-agent: MoodleBot
276Disallow: /
277
278User-agent: NeticleBot
279Disallow: /
280
281User-agent: Netvibes
282Disallow: /
283
284User-agent: PanguBot
285Disallow: /
286
287User-agent: PetalBot
288Disallow: /
289
290User-agent: SeekportBot
291Disallow: /
292
293User-agent: Seekr
294Disallow: /
295
296User-agent: SemrushBot
297Disallow: /
298
299User-agent: SeznamBot
300Disallow: /
301
302User-agent: Sidetrade Indexer Bot
303Disallow: /
304
305User-agent: Timpibot
306Disallow: /
307
308User-agent: Turnitin
309Disallow: /
310
311User-agent: VelenPublicWebCrawler
312Disallow: /
313
314User-agent: Webzio-Extended
315Disallow: /
316
317User-agent: yacy
318Disallow: /
319
320User-agent: yacybot
321Disallow: /
322
323User-agent: Yandex
324Disallow: /
325
326User-agent: YandexAdditional
327Disallow: /
328
329User-agent: YandexAdditionalBot
330Disallow: /
331
332User-agent: YandexBot
333Disallow: /
334
335User-agent: YandexImages
336Disallow: /
337
338User-agent: YandexNews
339Disallow: /
340
341User-agent: Baiduspider
342Disallow: /
343
344User-agent: Baiduspider-image
345Disallow: /
346
347User-agent: BaiduSpider-render
348Disallow: /
349
350User-agent: Baiduspider-video
351Disallow: /
352
353User-agent: Sogou Web Spider
354Disallow: /
355
356User-agent: yisouspider
357Disallow: /
358
359User-agent: YouBot
360Disallow: /
361
362User-agent: ia_archiver
363Disallow: /
364
365User-agent: archive.org_bot
366Disallow: /
367
368User-agent: ia_archiver-web.archive.org
369Disallow: /
370
371User-agent: aiHitBot
372Disallow: /
373
374User-agent: Amazon Kendra
375Disallow: /
376
377User-agent: Big Sur AI
378Disallow: /
379
380User-agent: Brandwatch
381Disallow: /
382
383User-agent: Bravest
384Disallow: /
385
386User-agent: ChatGPT Operator
387Disallow: /
388Allow: /branded-story/
389Allow: /contributor-content/
390Allow: /story/branded-story/
391Allow: /story/special/contributor-content/
392Allow: /money/money-management/
393Allow: /shopping/
394Allow: /story/shopping/
395Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/
396Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/
397Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/
398
399User-agent: ChatGPT-PageFetcher
400Disallow: /
401
402User-agent: OpenAI-CheckBot
403Disallow: /
404
405User-agent: OpenAI-Explore
406Disallow: /
407
408User-agent: OpenAI Feedback
409Disallow: /
410
411User-agent: OpenAI-SnippetBot
412Disallow: /
413
414User-agent: Cotoyogi
415Disallow: /
416
417User-agent: DigitalOceanGenAICrawler
418Disallow: /
419
420User-agent: Echobot Bot
421Disallow: /
422
423User-agent: EchoboxBot
424Disallow: /
425
426User-agent: Factset_spyderbot
427Disallow: /
428
429User-agent: Grok
430Disallow: /
431
432User-agent: iaskspider
433Disallow: /
434
435User-agent: imgproxy
436Disallow: /
437
438User-agent: LINER Bot
439Disallow: /
440
441User-agent: MistralAI-User
442Disallow: /
443
444User-agent: netEstate Imprint Crawler
445Disallow: /
446
447User-agent: NovaAct
448Disallow: /
449
450User-agent: QualifiedBot
451Disallow: /
452
453User-agent: SBIntuitionsBot
454Disallow: /
455
456User-agent: sbintuitionsbot
457Disallow: /
458
459User-agent: SemrushBot-OCOB
460Disallow: /
461
462User-agent: SemrushBotSwa
463Disallow: /
464
465User-agent: Sidetrade
466Disallow: /
467
468User-agent: TikTokSpider
469Disallow: /
470
471User-agent: WARDBot
472Disallow: /
473
474User-agent: TaraGroup Intelligent Bot V1
475Disallow: /
476
477User-agent: TaraGroup Intelligent Bot
478Disallow: /
479
480User-agent: DeepSeekBot
481Disallow: /
482
483User-agent: ChatGLM-Spider
484Disallow: /
485
486User-agent: GoogleOther
487Disallow: /
488
489User-agent: Google AI
490Disallow: /
491
492User-agent: Google-Agent
493Disallow: /
494
495User-agent: shapbot
496Disallow: /
497
498User-agent: FirecrawlAgent
499Disallow: /
500
501User-agent: Crawl4AI
502Disallow: /
503
504User-agent: imageSpider
505Disallow: /
506
507User-agent: laion-huggingface-processor
508Disallow: /
509
510User-agent: LCC
511Disallow: /
512
513User-agent: Spider
514Disallow: /
515
516User-agent: Datenbank Crawler
517Disallow: /
518
519User-agent: Cloudflare-AutoRAG
520Disallow: /
521
522User-agent: CloudflareBrowserRenderingCrawler
523Disallow: /
524
525User-agent: meta-webindexer
526Disallow: /
527
528User-agent: Bravebot
529Disallow: /
530
531User-agent: AddSearchBot
532Disallow: /
533
534User-agent: Anomura
535Disallow: /
536
537User-agent: atlassian-bot
538Disallow: /
539
540User-agent: Channel3Bot
541Disallow: /
542
543User-agent: LinkupBot
544Disallow: /
545
546User-agent: ZanistaBot
547Disallow: /
548
549User-agent: Gemini-Deep-Research
550Disallow: /
551
552User-agent: Google-NotebookLM
553Disallow: /
554
555User-agent: Perplexity-User
556Disallow: /
557
558User-agent: Perplexity Search Bot
559Disallow: /
560
561User-agent: Poetbot
562Disallow: /
563
564User-agent: Poseidon Research Crawler
565Disallow: /
566
567User-agent: ProRataInc
568Disallow: /
569
570User-agent: QuillBot
571Disallow: /
572
573User-agent: Tiingo News Agent
574Disallow: /
575
576User-agent: iboubot
577Disallow: /
578
579User-agent: TavilyBot
580Disallow: /
581
582User-agent: NeevaBot
583Disallow: /
584
585User-agent: PhindBot
586Disallow: /
587
588User-agent: AI2Bot-DeepResearchEval
589Disallow: /
590
591User-agent: KlaviyoAIBot
592Disallow: /
593
594User-agent: LinerBot
595Disallow: /
596
597User-agent: Poggio-Citations
598Disallow: /
599
600User-agent: Devin
601Disallow: /
602
603User-agent: bigsur.ai
604Disallow: /
605
606User-agent: ChatGPT Agent
607Disallow: /
608
609User-agent: GoogleAgent-Mariner
610Disallow: /
611
612User-agent: AmazonBuyForMe
613Disallow: /
614
615User-agent: Manus-User
616Disallow: /
617
618User-agent: manus-user
619Disallow: /
620
621User-agent: TwinAgent
622Disallow: /
623
624User-agent: AudigentAdBot
625Disallow: /
626
627User-agent: AudigentBot
628Disallow: /
629
630User-agent: JinaAI
631Disallow: /
632
633User-agent: Jina-AI
634Disallow: /
635
636User-agent: JinaBot
637Disallow: /
638
639User-agent: Jina-Reader
640Disallow: /
641
642User-agent: BrightData
643Disallow: /
644
645User-agent: Bright Data
646Disallow: /
647
648User-agent: BrightBot
649Disallow: /
650
651User-agent: SerpApi
652Disallow: /
653
654User-agent: SerpApiBot
655Disallow: /
656
657User-agent: Apify
658Disallow: /
659
660User-agent: ApifyBot
661Disallow: /
662
663User-agent: Apify-HTTP-Client
664Disallow: /
665
666User-agent: ScrapeGraphAI
667Disallow: /
668
669User-agent: scrapegraph
670Disallow: /
671
672User-agent: Scrapeless
673Disallow: /
674
675User-agent: ScrapelessBot
676Disallow: /
677
678User-agent: ZenRows
679Disallow: /
680
681User-agent: ZenRowsBot
682Disallow: /
683
684User-agent: PromptCloud
685Disallow: /
686
687User-agent: PromptCloudBot
688Disallow: /
689
690User-agent: Nimbleway
691Disallow: /
692
693User-agent: NimbleCrawler
694Disallow: /
695
696User-agent: NimblewayBot
697Disallow: /
698
699User-agent: Oxylabs
700Disallow: /
701
702User-agent: OxylabsBot
703Disallow: /
704
705User-agent: Zyte
706Disallow: /
707
708User-agent: ZyteBot
709Disallow: /
710
711User-agent: Scrapinghub
712Disallow: /
713
714User-agent: ScraperAPI
715Disallow: /
716
717User-agent: Scraper-API
718Disallow: /
719
720User-agent: ScrapingBee
721Disallow: /
722
723User-agent: ScrapingBeeBot
724Disallow: /
725
726User-agent: Scrapingdog
727Disallow: /
728
729User-agent: ScrapingdogBot
730Disallow: /
731
732User-agent: Grepsr
733Disallow: /
734
735User-agent: GrepsrBot
736Disallow: /
737
738User-agent: GrepsrCrawler
739Disallow: /
740
741User-agent: SpiderBot
742Disallow: /
743
744User-agent: spider-cloud
745Disallow: /
746
747User-agent: SpiderCloud
748Disallow: /
749
750User-agent: Hyperbrowser
751Disallow: /
752
753User-agent: HyperbrowserBot
754Disallow: /
755
756User-agent: HyperAgent
757Disallow: /
758
759User-agent: Scrapestack
760Disallow: /
761
762User-agent: ScrapestackBot
763Disallow: /
764
765User-agent: WebScraping.AI
766Disallow: /
767
768User-agent: WebScrapingAI
769Disallow: /
770
771User-agent: webscraping-ai
772Disallow: /
773
774User-agent: ForageAI
775Disallow: /
776
777User-agent: Forage-AI
778Disallow: /
779
780User-agent: ForageBot
781Disallow: /
782
783User-agent: DumplingAI
784Disallow: /
785
786User-agent: Dumpling-AI
787Disallow: /
788
789User-agent: DumplingBot
790Disallow: /
791
792User-agent: Import.io
793Disallow: /
794
795User-agent: ImportIO
796Disallow: /
797
798User-agent: importio
799Disallow: /
800
801User-agent: BrowseAI
802Disallow: /
803
804User-agent: Browse-AI
805Disallow: /
806
807User-agent: BrowseAIBot
808Disallow: /
809
810User-agent: Decodo
811Disallow: /
812
813User-agent: DecodoBot
814Disallow: /
815
816User-agent: Smartproxy
817Disallow: /
818
819User-agent: Scrapfly
820Disallow: /
821
822User-agent: ScrapflyBot
823Disallow: /
824
825User-agent: ScrapflyClient
826Disallow: /
827
828User-agent: Octoparse
829Disallow: /
830
831User-agent: OctoparseBot
832Disallow: /
833
834User-agent: TrajectData
835Disallow: /
836
837User-agent: Traject-Data
838Disallow: /
839
840User-agent: SerpWow
841Disallow: /
842
843User-agent: ScaleSERP
844Disallow: /
845
846User-agent: Rainforest-API
847Disallow: /
848
849User-agent: BlueCart-API
850Disallow: /
851
852User-agent: skrape.ai
853Disallow: /
854
855User-agent: SkrapeAI
856Disallow: /
857
858User-agent: SkrapeBot
859Disallow: /
860
861User-agent: Froxy
862Disallow: /
863
864User-agent: FroxyBot
865Disallow: /
866
867User-agent: MrScraper
868Disallow: /
869
870User-agent: Mr-Scraper
871Disallow: /
872
873User-agent: MrScraperBot
874Disallow: /
875
876User-agent: CrawlNow
877Disallow: /
878
879User-agent: CrawlNowBot
880Disallow: /
881
882User-agent: Soax
883Disallow: /
884
885User-agent: SoaxBot
886Disallow: /
887
888User-agent: SOAX
889Disallow: /
890
891User-agent: Kadoa
892Disallow: /
893
894User-agent: Kadoabot
895Disallow: /
896
897User-agent: KadoaBot
898Disallow: /
899
900User-agent: ScrapeHero
901Disallow: /
902
903User-agent: Scrape-Hero
904Disallow: /
905
906User-agent: ScrapeHeroBot
907Disallow: /
908
909User-agent: Exa
910Disallow: /
911
912User-agent: ExaBot
913Disallow: /
914
915User-agent: Exa-AI
916Disallow: /
917
918User-agent: ExaCrawler
919Disallow: /
920
921User-agent: Exabot
922Disallow: /
923
924User-agent: *
925Disallow: /errors
926Disallow: /interactive/
927Disallow: /userauth/
928Disallow: /ugc/
929Disallow: /feeds/
930Disallow: /services/
931Disallow: /facebook/
932Disallow: /version-info/
933Disallow: /longform/draft/
934Disallow: /story/draft/
935Disallow: /topic/*/smart/
936Disallow: /search
937Disallow: /module-showcase/
938Disallow: /newsletter/
939Disallow: /blended-newsletter/
940Disallow: /story/nletter/
941Disallow: /sports/services/photos/
942Disallow: /optimus
943Disallow: /ux-train
944Disallow: /story/advisory/
945Disallow: /.cam-tangent/
946Disallow: /pbd/
947Disallow: /gciaf/
948Disallow: /gcdn/gciaf/
949Disallow: /dcc/
950Disallow: /gcdn/dcc/
951Disallow: /dc/
952Disallow: /gcdn/dc/
953Disallow: /dcjs/
954Disallow: /gcdn/dcjs/
955Disallow: /content-queries/
956Disallow: /zxc/
957Disallow: /money/homefront/
958Disallow: /exp-cruise
959Disallow: /exp-las-vegas2
960Disallow: /exp-faw
961Disallow: /exp-caribbean
962Disallow: /exp-beach
963Disallow: /exp-cruise2
964Disallow: /yourtake
965Disallow: /story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/
966Disallow: /story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/
967Disallow: /story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/
968Disallow: /story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/
969Disallow: /money/lookup/stocks/
970Disallow: /money/blueprint/archive/
971Disallow: /money/blueprint/transfer?pname=
972
973User-agent: AmazonAdBot
974Allow: /
975
976User-agent: NewsBank.com
977Allow: /obituaries/
978
979Sitemap: https://www.usatoday.com/news-sitemap.xml
980Sitemap: https://www.usatoday.com/web-sitemap-index.xml
981Sitemap: https://www.usatoday.com/video-sitemap-index.xml
982Sitemap: https://www.usatoday.com/money/blueprint/sitemap.xml
983Sitemap: https://www.usatoday.com/money/blueprint/sitemap-news.xml
984Sitemap: https://www.usatoday.com/online-betting/sitemap.xml
985Sitemap: https://www.usatoday.com/online-betting/news-sitemap.xml