NeuralCrawl

The Atlantic / robots.txt snapshot

← back to theatlantic.com · fetched 2026-11-05T19:53:48Z (1d ago) · HTTP 200 · 5279 bytes · sha256 3f620aeca36c4630 · raw

final URL: https://www.theatlantic.com/robots.txt

1# As a condition of accessing this website, you agree to abide by the
2# following content signals:
3
4# (a) If a content-signal = yes, you may collect content for the
5# corresponding use.
6# (b) If a content-signal = no, you may not collect content for the
7# corresponding use.
8# (c) If the website operator does not include a content signal for a
9# corresponding use, the website operator neither grants nor restricts
10# permission via content signal with respect to the corresponding use.
11
12# The content signals and their meanings are:
13
14# search: building a search index and providing search results (e.g., returning
15# hyperlinks and short excerpts from your website's contents). Search
16# does not include providing AI-generated search summaries.
17# ai-input: inputting content into one or more AI models (e.g., retrieval
18# augmented generation, grounding, or other real-time taking of
19# content for generative AI search answers).
20# ai-train: training or fine-tuning AI models.
21
22# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
23# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
24# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
25
26User-agent: *
27Disallow: /magazine/archive/2010/11/letters-to-the-editor/308258/
28Disallow: /magazine/archive/2010/11/letters-to-the-editor/308258/*
29Disallow: /ab/*
30Disallow: /video/embed/
31Disallow: /zephr/*
32Disallow: /video/iframe/*
33Disallow: /search/?*q=*
34Allow: /magazine/archive/2001/02/bill-clinton-and-his-consequences/303383/$
35Disallow: /magazine/archive/2001/02/bill-clinton-and-his-consequences/303383/*
36Crawl-delay: 1
37Allow: /
38
39User-agent: AmazonAdBot
40Disallow: /ab/*
41Disallow: /video/embed/
42Disallow: /zephr/*
43Disallow: /video/iframe/*
44Disallow: /search/?*q=*
45Crawl-delay: 1
46Allow: /
47
48User-agent: Amazonbot
49Disallow: /
50Allow: /feed/*
51
52User-agent: anthropic-ai
53Disallow: /
54
55User-agent: Applebot
56Disallow: /ab/*
57Disallow: /video/embed/
58Disallow: /zephr/*
59Disallow: /video/iframe/*
60Disallow: /search/?*q=*
61Crawl-delay: 1
62Allow: /
63
64User-agent: Applebot-Extended
65Disallow: /
66
67User-agent: ArcSpanScraper
68Disallow: /ab/*
69Disallow: /video/embed/
70Disallow: /zephr/*
71Disallow: /video/iframe/*
72Disallow: /search/?*q=*
73Crawl-delay: 1
74Allow: /
75
76User-agent: AwarioRssBot
77User-agent: AwarioSmartBot
78Disallow: /
79
80User-agent: BingBot
81Content-signal: search=yes, ai-input=no, ai-train=no
82Disallow: /ab/*
83Disallow: /video/embed/
84Disallow: /zephr/*
85Disallow: /video/iframe/*
86Disallow: /search/?*q=*
87Crawl-delay: 1
88Allow: /
89
90User-agent: Bytespider
91Disallow: /
92
93User-agent: CCBot
94Disallow: /
95
96User-agent: ChatGPT-User
97Disallow: /ab/*
98Disallow: /video/embed/
99Disallow: /zephr/*
100Disallow: /video/iframe/*
101Disallow: /search/?*q=*
102Crawl-delay: 1
103Allow: /
104
105User-agent: ClaudeBot
106Disallow: /
107
108User-agent: Claude-SearchBot
109Allow: /sponsored/
110Disallow: /
111
112User-agent: Claude-User
113Allow: /sponsored/
114Disallow: /
115
116User-agent: Claude-Web
117Disallow: /
118
119User-agent: cohere-ai
120Disallow: /
121
122User-agent: DataForSeoBot
123Disallow: /
124
125User-agent: Diffbot
126Disallow: /
127
128User-agent: DuckAssistBot
129Content-signal: search=yes, ai-input=no, ai-train=no
130Disallow: /ab/*
131Disallow: /video/embed/
132Disallow: /zephr/*
133Disallow: /video/iframe/*
134Disallow: /search/?*q=*
135Crawl-delay: 1
136Allow: /
137
138User-agent: FacebookBot
139Content-signal: search=yes, ai-input=no, ai-train=no
140Disallow: /ab/*
141Disallow: /video/embed/
142Disallow: /zephr/*
143Disallow: /video/iframe/*
144Disallow: /search/?*q=*
145Crawl-delay: 1
146Allow: /
147
148User-agent: Google-Extended
149Disallow: /
150
151User-agent: Googlebot
152Content-signal: search=yes, ai-input=no, ai-train=no
153Disallow: /ab/*
154Disallow: /video/embed/
155Disallow: /zephr/*
156Disallow: /video/iframe/*
157Disallow: /search/?*q=*
158Crawl-delay: 1
159Allow: /
160
161User-agent: GPTBot
162Disallow: /ab/*
163Disallow: /video/embed/
164Disallow: /zephr/*
165Disallow: /video/iframe/*
166Disallow: /search/?*q=*
167Crawl-delay: 1
168Allow: /
169
170User-agent: magpie-crawler
171Disallow: /
172
173User-agent: Meta-ExternalAgent
174Disallow: /
175
176User-agent: Meta-ExternalFetcher
177Disallow: /
178
179User-agent: MistralAI-User
180Disallow: /
181
182User-agent: NewsNow
183Disallow: /
184
185User-agent: news-please
186Disallow: /
187
188User-agent: omgili
189Disallow: /
190
191User-agent: omgilibot
192Disallow: /
193
194User-agent: OAI-SearchBot
195Disallow: /ab/*
196Disallow: /video/embed/
197Disallow: /zephr/*
198Disallow: /video/iframe/*
199Disallow: /search/?*q=*
200Crawl-delay: 1
201Allow: /
202
203User-agent: Operator
204Disallow: /ab/*
205Disallow: /video/embed/
206Disallow: /zephr/*
207Disallow: /video/iframe/*
208Disallow: /search/?*q=*
209Crawl-delay: 1
210Allow: /
211
212User-agent: PetalBot
213Disallow: /
214
215User-agent: PerplexityBot
216Allow: /sponsored/
217Disallow: /
218
219User-agent: Perplexity-User
220Allow: /sponsored/
221Disallow: /
222
223User-agent: ProRataInc
224Disallow: /ab/*
225Disallow: /video/embed/
226Disallow: /zephr/*
227Disallow: /video/iframe/*
228Disallow: /search/?*q=*
229Crawl-delay: 1
230Allow: /
231
232User-agent: Quora-Bot
233Disallow: /
234
235User-agent: Scrapy
236Disallow: /
237
238User-agent: TerraCotta
239Disallow: /ab/*
240Disallow: /video/embed/
241Disallow: /zephr/*
242Disallow: /video/iframe/*
243Disallow: /search/?*q=*
244Crawl-delay: 1
245Allow: /
246
247User-agent: TimpiBot
248Disallow: /
249
250User-agent: TurnitinBot
251Disallow: /
252
253User-agent: archive.org_bot
254Disallow: /
255
256Sitemap: https://www.theatlantic.com/sitemap.xml
257Sitemap: https://www.theatlantic.com/sponsored/sitemap.xml