NeuralCrawl

The Atlantic / robots.txt snapshot

← back to theatlantic.com · fetched 2026-10-28T23:10:43Z (9d ago) · HTTP 200 · 5783 bytes · sha256 a87d80f198ff3a4d · raw

final URL: https://www.theatlantic.com/robots.txt

1# As a condition of accessing this website, you agree to abide by the
2# following content signals:
3
4# (a) If a content-signal = yes, you may collect content for the
5# corresponding use.
6# (b) If a content-signal = no, you may not collect content for the
7# corresponding use.
8# (c) If the website operator does not include a content signal for a
9# corresponding use, the website operator neither grants nor restricts
10# permission via content signal with respect to the corresponding use.
11
12# The content signals and their meanings are:
13
14# search: building a search index and providing search results (e.g., returning
15# hyperlinks and short excerpts from your website's contents). Search
16# does not include providing AI-generated search summaries.
17# ai-input: inputting content into one or more AI models (e.g., retrieval
18# augmented generation, grounding, or other real-time taking of
19# content for generative AI search answers).
20# ai-train: training or fine-tuning AI models.
21
22# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
23# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
24# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
25
26User-agent: *
27Disallow: /4624/TheAtlanticOnline/*
28Disallow: /magazine/archive/2010/11/letters-to-the-editor/308258/
29Disallow: /magazine/archive/2010/11/letters-to-the-editor/308258/*
30Disallow: /ab/*
31Disallow: /video/embed/
32Disallow: /zephr/*
33Disallow: /video/iframe/*
34Disallow: /search/?*q=*
35Allow: /magazine/archive/2001/02/bill-clinton-and-his-consequences/303383/$
36Disallow: /magazine/archive/2001/02/bill-clinton-and-his-consequences/303383/*
37Crawl-delay: 1
38Allow: /
39
40User-agent: AmazonAdBot
41Disallow: /4624/TheAtlanticOnline/*
42Disallow: /ab/*
43Disallow: /video/embed/
44Disallow: /zephr/*
45Disallow: /video/iframe/*
46Disallow: /search/?*q=*
47Crawl-delay: 1
48Allow: /
49
50User-agent: Amazonbot
51Disallow: /
52Allow: /feed/*
53
54User-agent: anthropic-ai
55Disallow: /
56
57User-agent: Applebot
58Disallow: /4624/TheAtlanticOnline/*
59Disallow: /ab/*
60Disallow: /video/embed/
61Disallow: /zephr/*
62Disallow: /video/iframe/*
63Disallow: /search/?*q=*
64Crawl-delay: 1
65Allow: /
66
67User-agent: Applebot-Extended
68Disallow: /
69
70User-agent: ArcSpanScraper
71Disallow: /4624/TheAtlanticOnline/*
72Disallow: /ab/*
73Disallow: /video/embed/
74Disallow: /zephr/*
75Disallow: /video/iframe/*
76Disallow: /search/?*q=*
77Crawl-delay: 1
78Allow: /
79
80User-agent: AwarioRssBot
81User-agent: AwarioSmartBot
82Disallow: /
83
84User-agent: BingBot
85Content-signal: search=yes, ai-input=no, ai-train=no
86Disallow: /4624/TheAtlanticOnline/*
87Disallow: /ab/*
88Disallow: /video/embed/
89Disallow: /zephr/*
90Disallow: /video/iframe/*
91Disallow: /search/?*q=*
92Crawl-delay: 1
93Allow: /
94
95User-agent: Bytespider
96Disallow: /
97
98User-agent: CCBot
99Disallow: /
100
101User-agent: ChatGPT-User
102Disallow: /4624/TheAtlanticOnline/*
103Disallow: /ab/*
104Disallow: /video/embed/
105Disallow: /zephr/*
106Disallow: /video/iframe/*
107Disallow: /search/?*q=*
108Crawl-delay: 1
109Allow: /
110
111User-agent: ClaudeBot
112Disallow: /
113
114User-agent: Claude-SearchBot
115Allow: /sponsored/
116Disallow: /
117
118User-agent: Claude-User
119Allow: /sponsored/
120Disallow: /
121
122User-agent: Claude-Web
123Disallow: /
124
125User-agent: cohere-ai
126Disallow: /
127
128User-agent: DataForSeoBot
129Disallow: /
130
131User-agent: Diffbot
132Disallow: /
133
134User-agent: DuckAssistBot
135Content-signal: search=yes, ai-input=no, ai-train=no
136Disallow: /4624/TheAtlanticOnline/*
137Disallow: /ab/*
138Disallow: /video/embed/
139Disallow: /zephr/*
140Disallow: /video/iframe/*
141Disallow: /search/?*q=*
142Crawl-delay: 1
143Allow: /
144
145User-agent: FacebookBot
146Content-signal: search=yes, ai-input=no, ai-train=no
147Disallow: /4624/TheAtlanticOnline/*
148Disallow: /ab/*
149Disallow: /video/embed/
150Disallow: /zephr/*
151Disallow: /video/iframe/*
152Disallow: /search/?*q=*
153Crawl-delay: 1
154Allow: /
155
156User-agent: Google-Extended
157Disallow: /
158
159User-agent: Googlebot
160Content-signal: search=yes, ai-input=no, ai-train=no
161Disallow: /4624/TheAtlanticOnline/*
162Disallow: /ab/*
163Disallow: /video/embed/
164Disallow: /zephr/*
165Disallow: /video/iframe/*
166Disallow: /search/?*q=*
167Crawl-delay: 1
168Allow: /
169
170User-agent: GPTBot
171Disallow: /4624/TheAtlanticOnline/*
172Disallow: /ab/*
173Disallow: /video/embed/
174Disallow: /zephr/*
175Disallow: /video/iframe/*
176Disallow: /search/?*q=*
177Crawl-delay: 1
178Allow: /
179
180User-agent: magpie-crawler
181Disallow: /
182
183User-agent: Meta-ExternalAgent
184Disallow: /
185
186User-agent: Meta-ExternalFetcher
187Disallow: /
188
189User-agent: MistralAI-User
190Disallow: /
191
192User-agent: NewsNow
193Disallow: /
194
195User-agent: news-please
196Disallow: /
197
198User-agent: omgili
199Disallow: /
200
201User-agent: omgilibot
202Disallow: /
203
204User-agent: OAI-SearchBot
205Disallow: /4624/TheAtlanticOnline/*
206Disallow: /ab/*
207Disallow: /video/embed/
208Disallow: /zephr/*
209Disallow: /video/iframe/*
210Disallow: /search/?*q=*
211Crawl-delay: 1
212Allow: /
213
214User-agent: Operator
215Disallow: /4624/TheAtlanticOnline/*
216Disallow: /ab/*
217Disallow: /video/embed/
218Disallow: /zephr/*
219Disallow: /video/iframe/*
220Disallow: /search/?*q=*
221Crawl-delay: 1
222Allow: /
223
224User-agent: PetalBot
225Disallow: /
226
227User-agent: PerplexityBot
228Allow: /sponsored/
229Disallow: /
230
231User-agent: Perplexity-User
232Allow: /sponsored/
233Disallow: /
234
235User-agent: ProRataInc
236Disallow: /4624/TheAtlanticOnline/*
237Disallow: /ab/*
238Disallow: /video/embed/
239Disallow: /zephr/*
240Disallow: /video/iframe/*
241Disallow: /search/?*q=*
242Crawl-delay: 1
243Allow: /
244
245User-agent: Quora-Bot
246Disallow: /
247
248User-agent: Scrapy
249Disallow: /
250
251User-agent: TerraCotta
252Disallow: /4624/TheAtlanticOnline/*
253Disallow: /ab/*
254Disallow: /video/embed/
255Disallow: /zephr/*
256Disallow: /video/iframe/*
257Disallow: /search/?*q=*
258Crawl-delay: 1
259Allow: /
260
261User-agent: TimpiBot
262Disallow: /
263
264User-agent: TurnitinBot
265Disallow: /
266
267User-agent: archive.org_bot
268Disallow: /
269
270Sitemap: https://www.theatlantic.com/sitemap.xml
271Sitemap: https://www.theatlantic.com/sponsored/sitemap.xml