NeuralCrawl

The New York Times / robots.txt snapshot

← back to nytimes.com · fetched 2026-06-20T01:10:30Z (1mo ago) · HTTP 200 · 8259 bytes · sha256 ebe60e191c7f2225 · raw

final URL: https://www.nytimes.com/robots.txt

1# New York Times content is made available for your personal, non-commercial
2# use subject to our Terms of Service here:
3# https://help.nytimes.com/hc/en-us/articles/115014893428-Terms-of-Service.
4# Use of any device, tool, or process designed to data mine or scrape the content
5# using automated means is prohibited without prior written permission from
6# The New York Times Company. Prohibited uses include but are not limited to:
7# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in
8# the Digital Single Market;
9# (2) the development of any software, machine learning, artificial intelligence (AI),
10# and/or large language models (LLMs);
11# (3) creating or providing archived or cached data sets containing our content to others; and/or
12# (4) any commercial purposes.
13# Contact https://nytlicensing.com/contact/ for assistance.
14
15User-agent: *
16User-agent: Googlebot
17Disallow: /ads/
18Disallow: /adx/bin/
19Disallow: /athletic/wp/wp-admin/
20Allow: /athletic/wp/wp-admin/admin-ajax.php
21Disallow: /athletic/async-*
22Disallow: /athletic/search/*
23Allow: /athletic/search/$
24Disallow: /athletic/checkout/
25Disallow: /athletic/checkout?plan_id*
26Allow: /athletic/checkout/$
27Disallow: /athletic/checkout2*
28Disallow: /athletic/login/
29Disallow: /athletic/login?login_source*
30Disallow: /athletic/login?ref_page*
31Allow: /athletic/login/$
32Disallow: /athletic/login2/
33Disallow: /athletic/login2?login_source*
34Disallow: /athletic/login2?ref_page*
35Allow: /athletic/login2/$
36Disallow: /athletic/report/
37Disallow: /athletic/*/discuss/*
38Disallow: /athletic/register/
39Disallow: /athletic/register?welcome_redirect*
40Disallow: /athletic/register2/
41Disallow: /athletic/register2?welcome_redirect*
42Disallow: /athletic/betmgm-redirect*
43Disallow: /athletic/cdn-cgi/
44Disallow: /athletic/verizon/*
45Disallow: /athletic/forgot-password/*
46Disallow: /athletic/forgot-password2/*
47Disallow: /athletic/amp-social-login*
48Disallow: /athletic/track-analytics/
49Disallow: /athletic/amp-auth/
50Disallow: /athletic/rss-feed/
51Disallow: /athletic/*?*rss=1
52Disallow: /athletic/global-color-test.php
53Disallow: /athletic/global-font-test.php
54Disallow: /athletic/graphql*
55Disallow: /athletic/api*
56Disallow: /athletic/ip*
57Disallow: /athletic/call-set-cookie-with-context/*
58Disallow: /athletic/get-current-user/
59Disallow: /athletic/pv.json
60Disallow: /athletic/following-feed-test/*
61Disallow: /athletic*/boxscore/*
62Disallow: /athletic/feed-test/
63Disallow: /athletic*/signed-mp3-redirect-url/*
64Disallow: /athletic/embedded-interactive/*
65Disallow: /card/panel/
66Disallow: /panel/
67Disallow: /puzzles/leaderboards/invite/*
68Disallow: /svc
69Allow: /svc/crosswords
70Allow: /svc/games
71Allow: /svc/letter-boxed
72Allow: /svc/spelling-bee
73Allow: /svc/wordle
74Allow: /svc/connections
75Allow: /svc/sudoku
76Allow: /svc/strands
77Allow: /svc/pips
78Disallow: /video/embedded/*
79Disallow: /search
80Disallow: /multiproduct/
81Disallow: /hd/
82Disallow: /inyt/
83Disallow: /*?*query=
84Disallow: /*.pdf$
85Disallow: /*?*login=
86Disallow: /*?*campaignId=
87Disallow: /*?*mcubz=
88Disallow: /*?*smprod=
89Disallow: /*?*ProfileID=
90Disallow: /*?*ListingID=
91Disallow: /*?*campaign_id=
92Disallow: /*?*hybrid=
93Disallow: /*?*entry=
94Disallow: /*?*embed=
95Disallow: /*?ls=
96Disallow: /*?*&ls=
97Disallow: /v1/initialize*
98Disallow: /v1/rgstr*
99Disallow: /wirecutter/wp-admin/
100Disallow: /wirecutter/*.zip$
101Disallow: /wirecutter/*.csv$
102Disallow: /wirecutter/deals/beta
103Disallow: /wirecutter/data-requests
104Disallow: /wirecutter/search
105Disallow: /wirecutter/*?s=
106Disallow: /wirecutter/*&xid=
107Disallow: /wirecutter/*?q=
108Disallow: /wirecutter/*?l=
109Disallow: /wirecutter/*?merchant=
110Disallow: /wirecutter/out/
111Disallow: /search
112Disallow: /subscription/*?*source=
113Disallow: /subscription/*?*onboarded=
114Disallow: /*?*smid=
115Disallow: /*?*partner=
116Disallow: /*?*utm_source=
117Allow: /wirecutter/*?*utm_source=
118Allow: /ads/public/
119Allow: /svc/news/v3/all/pshb.rss
120Allow: /wirecutter/reviews/*?*utm_source=
121Allow: /wirecutter/blog/*?*utm_source=
122Allow: /wirecutter/lists/*?*utm_source=
123Allow: /wirecutter/gifts/*?*utm_source=
124
125
126# Googlebot Specific Rules
127
128User-agent: Googlebot
129Disallow: /athletic*adgroupid*
130Disallow: /athletic*campaignid*
131Disallow: /athletic*ad_id*
132Disallow: /athletic*access_token*
133Disallow: /athletic*amp_reader_id*
134Disallow: /athletic*/?source=*
135Disallow: /athletic/*?*embed=1
136
137
138# Disallow Rules
139
140User-agent: AliyunSecBot
141Disallow: /
142
143User-agent: AmazonBot
144Disallow: /wirecutter/
145
146User-agent: anthropic-ai
147Disallow: /
148
149User-agent: Applebot-Extended
150Disallow: /
151
152User-agent: archive.org_bot
153Disallow: /
154
155User-agent: AudigentAdBot
156Disallow: /
157
158User-agent: AwarioRssBot
159User-agent: AwarioSmartBot
160Disallow: /
161
162User-agent: BLEXBot
163Disallow: /
164
165User-agent: Bytespider
166Disallow: /
167
168User-agent: CCBot
169Disallow: /
170
171User-agent: ChatGPT-User
172Disallow: /
173
174User-agent: ClaudeBot
175Disallow: /
176
177User-agent: Claude-SearchBot
178Disallow: /
179
180User-agent: Claude-User
181Disallow: /
182
183User-agent: Claude-Web
184Disallow: /
185
186User-agent: cohere-ai
187Disallow: /
188
189User-agent: DataForSeoBot
190Disallow: /
191
192User-agent: Diffbot
193Disallow: /
194
195User-agent: DuckAssistBot
196Disallow: /
197
198User-agent: EchoboxBot
199Disallow: /
200
201User-agent: FacebookBot
202Disallow: /
203
204User-agent: FriendlyCrawler
205Disallow: /
206
207User-agent: Google-CloudVertexBot
208Disallow: /
209Allow: /wirecutter/
210Allow: /athletic/
211
212User-agent: Google-Extended
213Disallow: /
214
215User-agent: GPTBot
216Disallow: /
217
218User-agent: ImagesiftBot
219Disallow: /
220
221User-agent: Jetslide
222Disallow: /
223
224User-agent: magpie-crawler
225Disallow: /
226
227User-agent: Meta-ExternalAgent
228User-agent: meta-externalagent
229Disallow: /
230
231User-agent: Meta-ExternalFetcher
232User-agent: meta-externalfetcher
233Disallow: /
234
235User-agent: Meta-WebIndexer
236User-agent: meta-webindexer/1.1
237Disallow: /
238
239User-agent: MyCentralAIScraperBot
240Disallow: /
241
242User-agent: NewsNow
243Disallow: /
244
245User-agent: news-please
246Disallow: /
247
248User-agent: OAI-SearchBot
249Disallow: /
250
251User-agent: omgili
252Disallow: /
253
254User-agent: omgilibot
255Disallow: /
256
257User-agent: peer39_crawler
258User-agent: peer39_crawler/1.0
259Disallow: /
260
261User-agent: PerplexityBot
262Disallow: /
263
264User-agent: Perplexity-User
265Disallow: /
266
267User-agent: Poseidon Research Crawler
268Disallow: /
269
270User-agent: quillbot.com
271Disallow: /
272
273User-agent: Quora-Bot
274Disallow: /
275
276User-agent: Scrapy
277Disallow: /
278
279User-agent: SeekrBot
280Disallow: /
281
282User-agent: SeznamHomepageCrawler
283Disallow: /
284
285User-agent: TaraGroup Intelligent Bot
286Disallow: /
287
288User-agent: Timpibot
289Disallow: /
290
291User-agent: TurnitinBot
292Disallow: /
293
294User-agent: ViennaTinyBot
295Disallow: /
296
297User-agent: YouBot
298Disallow: /
299
300
301# Ad Bot Rules
302
303User-agent: AmazonAdBot
304Allow: /
305
306
307# Social Bot Rules
308
309User-agent: facebookexternalhit
310Allow: /*?*smid=
311
312User-agent: Twitterbot
313Allow: /*?*smid=
314
315User-agent: RedditBot
316Allow: /*?*smid=
317
318# Sitemaps
319
320Sitemap: https://www.nytimes.com/sitemaps/new/news.xml.gz
321Sitemap: https://www.nytimes.com/sitemaps/new/sitemap.xml.gz
322Sitemap: https://www.nytimes.com/sitemaps/new/collections.xml.gz
323Sitemap: https://www.nytimes.com/sitemaps/new/video.xml.gz
324Sitemap: https://www.nytimes.com/sitemaps/new/cooking.xml.gz
325Sitemap: https://www.nytimes.com/sitemaps/new/recipe-collects.xml.gz
326Sitemap: https://www.nytimes.com/sitemaps/new/regions.xml
327Sitemap: https://www.nytimes.com/sitemaps/new/best-sellers.xml
328Sitemap: https://www.nytimes.com/sitemaps/new/subscription-landing-pages.xml
329Sitemap: https://www.nytimes.com/sitemaps/new/weather.xml.gz
330Sitemap: https://www.nytimes.com/sitemaps/new/espanol.xml.gz
331Sitemap: https://www.nytimes.com/sitemaps/new/espanol-collects.xml.gz
332Sitemap: https://www.nytimes.com/wirecutter/sitemapindex.xml
333Sitemap: https://www.nytimes.com/athletic/sitemap-videos-index.xml
334Sitemap: https://www.nytimes.com/athletic/sitemap-authors.xml
335Sitemap: https://www.nytimes.com/athletic/sitemap-verticals.xml
336Sitemap: https://www.nytimes.com/athletic/sitemap-teams.xml
337Sitemap: https://www.nytimes.com/athletic/sitemap-cities.xml
338Sitemap: https://www.nytimes.com/athletic/sitemap-players.xml
339Sitemap: https://www.nytimes.com/athletic/sitemap-tags.xml
340Sitemap: https://www.nytimes.com/athletic/sitemap-stats.xml
341Sitemap: https://www.nytimes.com/athletic/sitemap-schedule.xml
342Sitemap: https://www.nytimes.com/athletic/sitemap-roster.xml
343Sitemap: https://www.nytimes.com/athletic/sitemap.xml
344Sitemap: https://www.nytimes.com/games-assets/v2/assets/sitemap/games.xml