NeuralCrawl

The Verge / robots.txt snapshot

← back to theverge.com · fetched 2026-09-17T15:11:14Z (1d ago) · HTTP 200 · 5650 bytes · sha256 f74664eb7fc3cbdf · raw

final URL: https://theverge.com/robots.txt

1
2User-agent: Amazonbot
3Allow: /
4
5User-agent: Applebot
6Allow: /
7
8User-agent: GPTBot
9Allow: /
10
11User-agent: A6-Indexer
12Allow: /sp/
13Disallow: /
14
15User-agent: AhrefsBot
16Allow: /sp/
17Disallow: /
18
19User-agent: AI2Bot
20Allow: /sp/
21Disallow: /
22
23User-agent: AI2Bot-DeepResearchEval
24Allow: /sp/
25Disallow: /
26
27User-agent: Ai2Bot-Dolma
28Allow: /sp/
29Disallow: /
30
31User-agent: aiHitBot
32Allow: /sp/
33Disallow: /
34
35User-agent: Anchor Browser
36Allow: /sp/
37Disallow: /
38
39User-agent: anthropic-ai
40Allow: /sp/
41Disallow: /
42
43User-agent: ApifyBot
44Allow: /sp/
45Disallow: /
46
47User-agent: ApifyWebsiteContentCrawler
48Allow: /sp/
49Disallow: /
50
51User-agent: Applebot-Extended
52Allow: /sp/
53Disallow: /
54
55User-agent: Baiduspider
56Allow: /sp/
57Disallow: /
58
59User-agent: betaBot
60Allow: /sp/
61Disallow: /
62
63User-agent: BLEXBot
64Allow: /sp/
65Disallow: /
66
67User-agent: botify
68Allow: /sp/
69Disallow: /
70
71User-agent: brands-bot-logo
72Allow: /sp/
73Disallow: /
74
75User-agent: Brightbot
76Allow: /sp/
77Disallow: /
78
79User-agent: Bytespider
80Allow: /sp/
81Disallow: /
82
83User-agent: calculon
84Allow: /sp/
85Disallow: /
86
87User-agent: CCBot
88Allow: /sp/
89Disallow: /
90
91User-agent: ChatGLM-Spider
92Allow: /sp/
93Disallow: /
94
95User-agent: ChatGPT-User
96Allow: /sp/
97Disallow: /
98
99User-agent: Claude-SearchBot
100Allow: /sp/
101Disallow: /
102
103User-agent: Claude-User
104Allow: /sp/
105Disallow: /
106
107User-agent: Claude-Web
108Allow: /sp/
109Disallow: /
110
111User-agent: ClaudeBot
112Allow: /sp/
113Disallow: /
114
115User-agent: CloudVertexBot
116Allow: /sp/
117Disallow: /
118
119User-agent: cohere-ai
120Allow: /sp/
121Disallow: /
122
123User-agent: cohere-training-data-crawler
124Allow: /sp/
125Disallow: /
126
127User-agent: [email protected]
128Allow: /sp/
129Disallow: /
130
131User-agent: Cotoyogi
132Allow: /sp/
133Disallow: /
134
135User-agent: Crawl4AI
136Allow: /sp/
137Disallow: /
138
139User-agent: CrunchBot
140Allow: /sp/
141Disallow: /
142
143User-agent: deepcrawl
144Allow: /sp/
145Disallow: /
146
147User-agent: DeepSeekBot
148Allow: /sp/
149Disallow: /
150
151User-agent: Diffbot
152Allow: /sp/
153Disallow: /
154
155User-agent: DirBuster
156Allow: /sp/
157Disallow: /
158
159User-agent: DTS
160Allow: /sp/
161Disallow: /
162
163User-agent: DuckAssistBot
164Allow: /sp/
165Disallow: /
166
167User-agent: EasouSpider
168Allow: /sp/
169Disallow: /
170
171User-agent: FacebookBot
172Allow: /sp/
173Disallow: /
174
175User-agent: FirecrawlAgent
176Allow: /sp/
177Disallow: /
178
179User-agent: FriendlyCrawler
180Allow: /sp/
181Disallow: /
182
183User-agent: GlutenCrawler
184Allow: /sp/
185Disallow: /
186
187User-agent: Google-CloudVertexBot
188Allow: /sp/
189Disallow: /
190
191User-agent: Google-Extended
192Allow: /sp/
193Disallow: /
194
195User-agent: GrapeshotCrawler
196Allow: /sp/
197Disallow: /
198
199User-agent: iaskspider
200Allow: /sp/
201Disallow: /
202
203User-agent: ICC-Crawler
204Allow: /sp/
205Disallow: /
206
207User-agent: ImagesiftBot
208Allow: /sp/
209Disallow: /
210
211User-agent: img2dataset
212Allow: /sp/
213Disallow: /
214
215User-agent: JackCrawl
216Allow: /sp/
217Disallow: /
218
219User-agent: Jetslide
220Allow: /sp/
221Disallow: /
222
223User-agent: linkapedia_bot
224Allow: /sp/
225Disallow: /
226
227User-agent: Mediatoolkitbot
228Allow: /sp/
229Disallow: /
230
231User-agent: MegaIndex
232Allow: /sp/
233Disallow: /
234
235User-agent: meta-externalagent
236Allow: /sp/
237Disallow: /
238
239User-agent: meta-externalfetcher
240Allow: /sp/
241Disallow: /
242
243User-agent: Microsoft BITS
244Allow: /sp/
245Disallow: /
246
247User-agent: Microsoft URL Control
248Allow: /sp/
249Disallow: /
250
251User-agent: Microsoft-WebDAV-MiniRedir
252Allow: /sp/
253Disallow: /
254
255User-agent: MistralAI-User
256Allow: /sp/
257Disallow: /
258
259User-agent: news-please
260Allow: /sp/
261Disallow: /
262
263User-agent: Novellum
264Allow: /sp/
265Disallow: /
266
267User-agent: NTENTbot
268Allow: /sp/
269Disallow: /
270
271User-agent: Nutch
272Allow: /sp/
273Disallow: /
274
275User-agent: omgili
276Allow: /sp/
277Disallow: /
278
279User-agent: Omgilibot
280Allow: /sp/
281Disallow: /
282
283User-agent: OpenLinkProfiler
284Allow: /sp/
285Disallow: /
286
287User-agent: Perplexity-User
288Allow: /sp/
289Disallow: /
290
291User-agent: PerplexityBot
292Allow: /sp/
293Disallow: /
294
295User-agent: PetalBot
296Allow: /sp/
297Disallow: /
298
299User-agent: Primalbot
300Allow: /sp/
301Disallow: /
302
303User-agent: python-requests
304Allow: /sp/
305Disallow: /
306
307User-agent: Re-Animator
308Allow: /sp/
309Disallow: /
310
311User-agent: Riddler
312Allow: /sp/
313Disallow: /
314
315User-agent: SafeDNSBot
316Allow: /sp/
317Disallow: /
318
319User-agent: SBIntuitionsBot
320Allow: /sp/
321Disallow: /
322
323User-agent: Scrapy
324Allow: /sp/
325Disallow: /
326
327User-agent: SearchAtlas
328Allow: /sp/
329Disallow: /
330
331User-agent: SemanticScholarBot
332Allow: /sp/
333Disallow: /
334
335User-agent: SemrushBot
336Allow: /sp/
337Disallow: /
338
339User-agent: SentiBot
340Allow: /sp/
341Disallow: /
342
343User-agent: Sogou
344Allow: /sp/
345Disallow: /
346
347User-agent: Spider-Pig
348Allow: /sp/
349Disallow: /
350
351User-agent: sqlmap
352Allow: /sp/
353Disallow: /
354
355User-agent: StorygizeBot
356Allow: /sp/
357Disallow: /
358
359User-agent: Test Certificate Info
360Allow: /sp/
361Disallow: /
362
363User-agent: TikTokSpider
364Allow: /sp/
365Disallow: /
366
367User-agent: Timpibot
368Allow: /sp/
369Disallow: /
370
371User-agent: Turnitin
372Allow: /sp/
373Disallow: /
374
375User-agent: uipbot
376Allow: /sp/
377Disallow: /
378
379User-agent: vBSEO
380Allow: /sp/
381Disallow: /
382
383User-agent: VegeBot
384Allow: /sp/
385Disallow: /
386
387User-agent: VelenPublicWebCrawler
388Allow: /sp/
389Disallow: /
390
391User-agent: Web Link Validator
392Allow: /sp/
393Disallow: /
394
395User-agent: WebCruiser
396Allow: /sp/
397Disallow: /
398
399User-agent: webzio-extended
400Allow: /sp/
401Disallow: /
402
403User-agent: Xenu
404Allow: /sp/
405Disallow: /
406
407User-agent: YouBot
408Allow: /sp/
409Disallow: /
410
411User-agent: *
412Disallow: /wp-admin/
413Allow: /wp-admin/admin-ajax.php
414
415User-agent: Googlebot-News
416Disallow: /ad/
417Disallow: /sponsored/
418Disallow: /sp/
419
420User-agent: *
421Allow: /sp/
422Disallow: /admin
423Disallow: /newfanshot
424Disallow: /users/*/replies
425Disallow: /users/*/comments
426Disallow: /login
427Disallow: /account
428Disallow: /auth/*
429Disallow: /chorus_auth
430Disallow: /sso
431Disallow: /search
432Disallow: /the-highlight$
433Disallow: /share$
434Disallow: /share/*
435Disallow: /share?*
436
437# Google news sitemap
438Sitemap: https://theverge.com/sitemaps/google_news
439
440# Sitemap archive
441Sitemap: https://theverge.com/sitemaps