NeuralCrawl

Vox / robots.txt snapshot

← back to vox.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 5648 bytes · sha256 4043372683e2f48d · raw

final URL: https://www.vox.com/robots.txt

1
2User-agent: Amazonbot
3Allow: /
4
5User-agent: Applebot
6Allow: /
7
8User-agent: GPTBot
9Allow: /
10
11User-agent: A6-Indexer
12Disallow: /
13Allow: /sp/
14
15User-agent: AhrefsBot
16Disallow: /
17Allow: /sp/
18
19User-agent: AI2Bot
20Disallow: /
21Allow: /sp/
22
23User-agent: AI2Bot-DeepResearchEval
24Disallow: /
25Allow: /sp/
26
27User-agent: Ai2Bot-Dolma
28Disallow: /
29Allow: /sp/
30
31User-agent: aiHitBot
32Disallow: /
33Allow: /sp/
34
35User-agent: Anchor Browser
36Disallow: /
37Allow: /sp/
38
39User-agent: anthropic-ai
40Disallow: /
41Allow: /sp/
42
43User-agent: ApifyBot
44Disallow: /
45Allow: /sp/
46
47User-agent: ApifyWebsiteContentCrawler
48Disallow: /
49Allow: /sp/
50
51User-agent: Applebot-Extended
52Disallow: /
53Allow: /sp/
54
55User-agent: Baiduspider
56Disallow: /
57Allow: /sp/
58
59User-agent: betaBot
60Disallow: /
61Allow: /sp/
62
63User-agent: BLEXBot
64Disallow: /
65Allow: /sp/
66
67User-agent: botify
68Disallow: /
69Allow: /sp/
70
71User-agent: brands-bot-logo
72Disallow: /
73Allow: /sp/
74
75User-agent: Brightbot
76Disallow: /
77Allow: /sp/
78
79User-agent: Bytespider
80Disallow: /
81Allow: /sp/
82
83User-agent: calculon
84Disallow: /
85Allow: /sp/
86
87User-agent: CCBot
88Disallow: /
89Allow: /sp/
90
91User-agent: ChatGLM-Spider
92Disallow: /
93Allow: /sp/
94
95User-agent: ChatGPT-User
96Disallow: /
97Allow: /sp/
98
99User-agent: Claude-SearchBot
100Disallow: /
101Allow: /sp/
102
103User-agent: Claude-User
104Disallow: /
105Allow: /sp/
106
107User-agent: Claude-Web
108Disallow: /
109Allow: /sp/
110
111User-agent: ClaudeBot
112Disallow: /
113Allow: /sp/
114
115User-agent: CloudVertexBot
116Disallow: /
117Allow: /sp/
118
119User-agent: cohere-ai
120Disallow: /
121Allow: /sp/
122
123User-agent: cohere-training-data-crawler
124Disallow: /
125Allow: /sp/
126
127User-agent: [email protected]
128Disallow: /
129Allow: /sp/
130
131User-agent: Cotoyogi
132Disallow: /
133Allow: /sp/
134
135User-agent: Crawl4AI
136Disallow: /
137Allow: /sp/
138
139User-agent: CrunchBot
140Disallow: /
141Allow: /sp/
142
143User-agent: deepcrawl
144Disallow: /
145Allow: /sp/
146
147User-agent: DeepSeekBot
148Disallow: /
149Allow: /sp/
150
151User-agent: Diffbot
152Disallow: /
153Allow: /sp/
154
155User-agent: DirBuster
156Disallow: /
157Allow: /sp/
158
159User-agent: DTS
160Disallow: /
161Allow: /sp/
162
163User-agent: DuckAssistBot
164Disallow: /
165Allow: /sp/
166
167User-agent: EasouSpider
168Disallow: /
169Allow: /sp/
170
171User-agent: FacebookBot
172Disallow: /
173Allow: /sp/
174
175User-agent: FirecrawlAgent
176Disallow: /
177Allow: /sp/
178
179User-agent: FriendlyCrawler
180Disallow: /
181Allow: /sp/
182
183User-agent: GlutenCrawler
184Disallow: /
185Allow: /sp/
186
187User-agent: Google-CloudVertexBot
188Disallow: /
189Allow: /sp/
190
191User-agent: Google-Extended
192Disallow: /
193Allow: /sp/
194
195User-agent: GrapeshotCrawler
196Disallow: /
197Allow: /sp/
198
199User-agent: iaskspider
200Disallow: /
201Allow: /sp/
202
203User-agent: ICC-Crawler
204Disallow: /
205Allow: /sp/
206
207User-agent: ImagesiftBot
208Disallow: /
209Allow: /sp/
210
211User-agent: img2dataset
212Disallow: /
213Allow: /sp/
214
215User-agent: JackCrawl
216Disallow: /
217Allow: /sp/
218
219User-agent: Jetslide
220Disallow: /
221Allow: /sp/
222
223User-agent: linkapedia_bot
224Disallow: /
225Allow: /sp/
226
227User-agent: Mediatoolkitbot
228Disallow: /
229Allow: /sp/
230
231User-agent: MegaIndex
232Disallow: /
233Allow: /sp/
234
235User-agent: meta-externalagent
236Disallow: /
237Allow: /sp/
238
239User-agent: meta-externalfetcher
240Disallow: /
241Allow: /sp/
242
243User-agent: Microsoft BITS
244Disallow: /
245Allow: /sp/
246
247User-agent: Microsoft URL Control
248Disallow: /
249Allow: /sp/
250
251User-agent: Microsoft-WebDAV-MiniRedir
252Disallow: /
253Allow: /sp/
254
255User-agent: MistralAI-User
256Disallow: /
257Allow: /sp/
258
259User-agent: news-please
260Disallow: /
261Allow: /sp/
262
263User-agent: Novellum
264Disallow: /
265Allow: /sp/
266
267User-agent: NTENTbot
268Disallow: /
269Allow: /sp/
270
271User-agent: Nutch
272Disallow: /
273Allow: /sp/
274
275User-agent: omgili
276Disallow: /
277Allow: /sp/
278
279User-agent: Omgilibot
280Disallow: /
281Allow: /sp/
282
283User-agent: OpenLinkProfiler
284Disallow: /
285Allow: /sp/
286
287User-agent: Perplexity-User
288Disallow: /
289Allow: /sp/
290
291User-agent: PerplexityBot
292Disallow: /
293Allow: /sp/
294
295User-agent: PetalBot
296Disallow: /
297Allow: /sp/
298
299User-agent: Primalbot
300Disallow: /
301Allow: /sp/
302
303User-agent: python-requests
304Disallow: /
305Allow: /sp/
306
307User-agent: Re-Animator
308Disallow: /
309Allow: /sp/
310
311User-agent: Riddler
312Disallow: /
313Allow: /sp/
314
315User-agent: SafeDNSBot
316Disallow: /
317Allow: /sp/
318
319User-agent: SBIntuitionsBot
320Disallow: /
321Allow: /sp/
322
323User-agent: Scrapy
324Disallow: /
325Allow: /sp/
326
327User-agent: SearchAtlas
328Disallow: /
329Allow: /sp/
330
331User-agent: SemanticScholarBot
332Disallow: /
333Allow: /sp/
334
335User-agent: SemrushBot
336Disallow: /
337Allow: /sp/
338
339User-agent: SentiBot
340Disallow: /
341Allow: /sp/
342
343User-agent: Sogou
344Disallow: /
345Allow: /sp/
346
347User-agent: Spider-Pig
348Disallow: /
349Allow: /sp/
350
351User-agent: sqlmap
352Disallow: /
353Allow: /sp/
354
355User-agent: StorygizeBot
356Disallow: /
357Allow: /sp/
358
359User-agent: Test Certificate Info
360Disallow: /
361Allow: /sp/
362
363User-agent: TikTokSpider
364Disallow: /
365Allow: /sp/
366
367User-agent: Timpibot
368Disallow: /
369Allow: /sp/
370
371User-agent: Turnitin
372Disallow: /
373Allow: /sp/
374
375User-agent: uipbot
376Disallow: /
377Allow: /sp/
378
379User-agent: vBSEO
380Disallow: /
381Allow: /sp/
382
383User-agent: VegeBot
384Disallow: /
385Allow: /sp/
386
387User-agent: VelenPublicWebCrawler
388Disallow: /
389Allow: /sp/
390
391User-agent: Web Link Validator
392Disallow: /
393Allow: /sp/
394
395User-agent: WebCruiser
396Disallow: /
397Allow: /sp/
398
399User-agent: webzio-extended
400Disallow: /
401Allow: /sp/
402
403User-agent: Xenu
404Disallow: /
405Allow: /sp/
406
407User-agent: YouBot
408Disallow: /
409Allow: /sp/
410
411User-agent: *
412Disallow: /wp-admin/
413Allow: /wp-admin/admin-ajax.php
414
415User-agent: Googlebot-News
416Disallow: /ad/
417Disallow: /sponsored/
418Disallow: /sp/
419
420User-agent: *
421Allow: /sp/
422Disallow: /admin
423Disallow: /newfanshot
424Disallow: /users/*/replies
425Disallow: /users/*/comments
426Disallow: /login
427Disallow: /account
428Disallow: /auth/*
429Disallow: /chorus_auth
430Disallow: /sso
431Disallow: /search
432Disallow: /the-highlight$
433Disallow: /share$
434Disallow: /share/*
435Disallow: /share?*
436
437# Google news sitemap
438Sitemap: https://www.vox.com/sitemaps/google_news
439
440# Sitemap archive
441Sitemap: https://www.vox.com/sitemaps