NeuralCrawl

The New York Times / robots.txt snapshot

← back to nytimes.com · fetched 2026-08-27T20:19:03Z (22d ago) · HTTP 200 · 8475 bytes · sha256 bde4135fae277972 · raw

final URL: https://www.nytimes.com/robots.txt

1# New York Times content is made available for your personal, non-commercial
2# use subject to our Terms of Service here:
3# https://help.nytimes.com/hc/en-us/articles/115014893428-Terms-of-Service.
4# Use of any device, tool, or process designed to data mine or scrape the content
5# using automated means is prohibited without prior written permission from
6# The New York Times Company. Prohibited uses include but are not limited to:
7# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in
8# the Digital Single Market;
9# (2) the development of any software, machine learning, artificial intelligence (AI),
10# and/or large language models (LLMs);
11# (3) creating or providing archived or cached data sets containing our content to others; and/or
12# (4) any commercial purposes.
13# Contact https://nytlicensing.com/contact/ for assistance.
14
15User-agent: *
16User-agent: Googlebot
17Disallow: /ads/
18Disallow: /adx/bin/
19Disallow: /athletic/wp/wp-admin/
20Allow: /athletic/wp/wp-admin/admin-ajax.php
21Disallow: /athletic/async-*
22Disallow: /athletic/search/*
23Allow: /athletic/search/$
24Disallow: /athletic/checkout/
25Disallow: /athletic/checkout?plan_id*
26Allow: /athletic/checkout/$
27Disallow: /athletic/checkout2*
28Disallow: /athletic/login/
29Disallow: /athletic/login?login_source*
30Disallow: /athletic/login?ref_page*
31Allow: /athletic/login/$
32Disallow: /athletic/login2/
33Disallow: /athletic/login2?login_source*
34Disallow: /athletic/login2?ref_page*
35Allow: /athletic/login2/$
36Disallow: /athletic/report/
37Disallow: /athletic/*/discuss/*
38Disallow: /athletic/register/
39Disallow: /athletic/register?welcome_redirect*
40Disallow: /athletic/register2/
41Disallow: /athletic/register2?welcome_redirect*
42Disallow: /athletic/betmgm-redirect*
43Disallow: /athletic/cdn-cgi/
44Disallow: /athletic/verizon/*
45Disallow: /athletic/forgot-password/*
46Disallow: /athletic/forgot-password2/*
47Disallow: /athletic/amp-social-login*
48Disallow: /athletic/track-analytics/
49Disallow: /athletic/amp-auth/
50Disallow: /athletic/rss-feed/
51Disallow: /athletic/*?*rss=1
52Disallow: /athletic/global-color-test.php
53Disallow: /athletic/global-font-test.php
54Disallow: /athletic/graphql*
55Disallow: /athletic/api*
56Disallow: /athletic/ip*
57Disallow: /athletic/call-set-cookie-with-context/*
58Disallow: /athletic/get-current-user/
59Disallow: /athletic/pv.json
60Disallow: /athletic/following-feed-test/*
61Disallow: /athletic*/boxscore/*
62Disallow: /athletic/feed-test/
63Disallow: /athletic*/signed-mp3-redirect-url/*
64Disallow: /athletic/embedded-interactive/*
65Disallow: /card/panel/
66Disallow: /panel/
67Disallow: /puzzles/leaderboards/invite/*
68Disallow: /svc
69Allow: /svc/crosswords
70Allow: /svc/games
71Allow: /svc/letter-boxed
72Allow: /svc/spelling-bee
73Allow: /svc/wordle
74Allow: /svc/connections
75Allow: /svc/sudoku
76Allow: /svc/strands
77Allow: /svc/pips
78Disallow: /video/embedded/*
79Disallow: /search
80Disallow: /multiproduct/
81Disallow: /hd/
82Disallow: /inyt/
83Disallow: /*?*query=
84Disallow: /*.pdf$
85Disallow: /*?*login=
86Disallow: /*?*campaignId=
87Disallow: /*?*mcubz=
88Disallow: /*?*smprod=
89Disallow: /*?*ProfileID=
90Disallow: /*?*ListingID=
91Disallow: /*?*campaign_id=
92Disallow: /*?*hybrid=
93Disallow: /*?*entry=
94Disallow: /*?*embed=
95Disallow: /*?ls=
96Disallow: /*?*&ls=
97Disallow: /v1/initialize*
98Disallow: /v1/rgstr*
99Disallow: /wirecutter/wp-admin/
100Disallow: /wirecutter/*.zip$
101Disallow: /wirecutter/*.csv$
102Disallow: /wirecutter/deals/beta
103Disallow: /wirecutter/data-requests
104Disallow: /wirecutter/search
105Disallow: /wirecutter/*?s=
106Disallow: /wirecutter/*&xid=
107Disallow: /wirecutter/*?q=
108Disallow: /wirecutter/*?l=
109Disallow: /wirecutter/*?merchant=
110Disallow: /wirecutter/out/
111Disallow: /search
112Disallow: /subscription/*?*source=
113Disallow: /subscription/*?*onboarded=
114Disallow: /*?*smid=
115Disallow: /*?*partner=
116Disallow: /*?*utm_source=
117Allow: /wirecutter/*?*utm_source=
118Allow: /ads/public/
119Allow: /svc/news/v3/all/pshb.rss
120Allow: /wirecutter/reviews/*?*utm_source=
121Allow: /wirecutter/blog/*?*utm_source=
122Allow: /wirecutter/lists/*?*utm_source=
123Allow: /wirecutter/gifts/*?*utm_source=
124
125
126# Googlebot Specific Rules
127
128User-agent: Googlebot
129Disallow: /athletic*adgroupid*
130Disallow: /athletic*campaignid*
131Disallow: /athletic*ad_id*
132Disallow: /athletic*access_token*
133Disallow: /athletic*amp_reader_id*
134Disallow: /athletic*/?source=*
135Disallow: /athletic/*?*embed=1
136
137
138# Disallow Rules
139
140User-agent: AliyunSecBot
141Disallow: /
142
143User-agent: Amazonbot
144Disallow: /wirecutter/
145
146User-agent: anthropic-ai
147Disallow: /
148
149User-agent: Applebot-Extended
150Disallow: /
151
152User-agent: archive.org_bot
153Disallow: /
154
155User-agent: AudigentAdBot
156Disallow: /
157
158User-agent: AwarioRssBot
159User-agent: AwarioSmartBot
160Disallow: /
161
162User-agent: BLEXBot
163Disallow: /
164
165User-agent: Brightbot
166Disallow: /
167
168User-agent: Bytespider
169Disallow: /
170
171User-agent: CCBot
172Disallow: /
173
174User-agent: ChatGPT-User
175Disallow: /
176
177User-agent: ClaudeBot
178Disallow: /
179
180User-agent: Claude-SearchBot
181Disallow: /
182
183User-agent: Claude-User
184Disallow: /
185
186User-agent: Claude-Web
187Disallow: /
188
189User-agent: cohere-ai
190Disallow: /
191
192User-agent: DataForSeoBot
193Disallow: /
194
195User-agent: Diffbot
196Disallow: /
197
198User-agent: Diffbot-User
199Disallow: /
200
201User-agent: DuckAssistBot
202Disallow: /
203
204User-agent: EchoboxBot
205Disallow: /
206
207User-agent: ExaBot
208Disallow: /
209
210User-agent: ExaSearchBot
211Disallow: /
212
213User-agent: FacebookBot
214Disallow: /
215
216User-agent: FirecrawlAgent
217Disallow: /
218
219User-agent: FriendlyCrawler
220Disallow: /
221
222User-agent: Google-CloudVertexBot
223Disallow: /
224Allow: /wirecutter/
225Allow: /athletic/
226
227User-agent: Google-Extended
228Disallow: /
229
230User-agent: GPTBot
231Disallow: /
232
233User-agent: ImagesiftBot
234Disallow: /
235
236User-agent: Jetslide
237Disallow: /
238
239User-agent: magpie-crawler
240Disallow: /
241
242User-agent: Meta-ExternalAgent
243User-agent: meta-externalagent
244Disallow: /
245
246User-agent: Meta-ExternalFetcher
247User-agent: meta-externalfetcher
248Disallow: /
249
250User-agent: Meta-WebIndexer
251User-agent: meta-webindexer/1.1
252Disallow: /
253
254User-agent: MyCentralAIScraperBot
255Disallow: /
256
257User-agent: NewsNow
258Disallow: /
259
260User-agent: news-please
261Disallow: /
262
263User-agent: OAI-SearchBot
264Disallow: /
265
266User-agent: omgili
267Disallow: /
268
269User-agent: omgilibot
270Disallow: /
271
272User-agent: peer39_crawler
273User-agent: peer39_crawler/1.0
274Disallow: /
275
276User-agent: PerplexityBot
277Disallow: /
278
279User-agent: Perplexity-User
280Disallow: /
281
282User-agent: Poseidon Research Crawler
283Disallow: /
284
285User-agent: quillbot.com
286Disallow: /
287
288User-agent: Quora-Bot
289Disallow: /
290
291User-agent: Scrapy
292Disallow: /
293
294User-agent: SeekrBot
295Disallow: /
296
297User-agent: SeznamHomepageCrawler
298Disallow: /
299
300User-agent: ShapBot
301Disallow: /
302
303User-agent: TaraGroup Intelligent Bot
304Disallow: /
305
306User-agent: Timpibot
307Disallow: /
308
309User-agent: TurnitinBot
310Disallow: /
311
312User-agent: ViennaTinyBot
313Disallow: /
314
315User-agent: YouBot
316Disallow: /
317
318
319# Ad Bot Rules
320
321User-agent: AmazonAdBot
322Allow: /
323
324
325# Social Bot Rules
326
327User-agent: facebookexternalhit
328Allow: /*?*smid=
329
330User-agent: Twitterbot
331Allow: /*?*smid=
332
333User-agent: RedditBot
334Allow: /*?*smid=
335
336# Sitemaps
337
338Sitemap: https://www.nytimes.com/sitemaps/new/news.xml.gz
339Sitemap: https://www.nytimes.com/sitemaps/new/sitemap.xml.gz
340Sitemap: https://www.nytimes.com/sitemaps/new/collections.xml.gz
341Sitemap: https://www.nytimes.com/sitemaps/new/video.xml.gz
342Sitemap: https://www.nytimes.com/sitemaps/new/cooking.xml.gz
343Sitemap: https://www.nytimes.com/sitemaps/new/recipe-collects.xml.gz
344Sitemap: https://www.nytimes.com/sitemaps/new/regions.xml
345Sitemap: https://www.nytimes.com/sitemaps/new/best-sellers.xml
346Sitemap: https://www.nytimes.com/sitemaps/new/subscription-landing-pages.xml
347Sitemap: https://www.nytimes.com/sitemaps/new/weather.xml.gz
348Sitemap: https://www.nytimes.com/sitemaps/new/espanol.xml.gz
349Sitemap: https://www.nytimes.com/sitemaps/new/espanol-collects.xml.gz
350Sitemap: https://www.nytimes.com/wirecutter/sitemapindex.xml
351Sitemap: https://www.nytimes.com/athletic/sitemap-videos-index.xml
352Sitemap: https://www.nytimes.com/athletic/sitemap-authors.xml
353Sitemap: https://www.nytimes.com/athletic/sitemap-verticals.xml
354Sitemap: https://www.nytimes.com/athletic/sitemap-teams.xml
355Sitemap: https://www.nytimes.com/athletic/sitemap-cities.xml
356Sitemap: https://www.nytimes.com/athletic/sitemap-players.xml
357Sitemap: https://www.nytimes.com/athletic/sitemap-tags.xml
358Sitemap: https://www.nytimes.com/athletic/sitemap-stats.xml
359Sitemap: https://www.nytimes.com/athletic/sitemap-schedule.xml
360Sitemap: https://www.nytimes.com/athletic/sitemap-roster.xml
361Sitemap: https://www.nytimes.com/athletic/sitemap.xml
362Sitemap: https://www.nytimes.com/games-assets/v2/assets/sitemap/games.xml