NeuralCrawl

El País / robots.txt snapshot

← back to elpais.com · fetched 2026-06-24T22:21:31Z (1d ago) · HTTP 200 · 7712 bytes · sha256 5af61f62959c2df1 · raw

final URL: https://elpais.com/robots.txt

1User-agent: PetalBot
2Disallow: /
3
4# Bloque bots global confirmado 20260505
5User-agent: bedrock-agentcore-browser
6Disallow: /
7
8User-agent: Amazon-Bot
9Disallow: /
10
11User-agent: Amazonbot
12Disallow: /
13
14User-agent: AmazonProductDiscoveryBot
15Disallow: /
16
17User-agent: Amzn-SearchBot
18Disallow: /
19
20User-agent: Amzn-User
21Disallow: /
22
23User-agent: Arquivo-web-crawler
24Disallow: /
25
26User-agent: bnf.fr_bot
27Disallow: /
28
29User-agent: Bytespider
30Disallow: /
31
32User-agent: Claude-Extension
33Disallow: /
34
35User-agent: ClaudeBot
36Disallow: /
37
38User-agent: Claude-User
39Disallow: /
40
41User-agent: Claude-SearchBot
42Disallow: /
43
44User-agent: Comet
45Disallow: /
46
47User-agent: CloudFlare-AlwaysOnline
48Disallow: /
49
50User-agent: Cloudflare-Healthchecks
51Disallow: /
52
53User-agent: CloudflareBrowserRendering
54Disallow: /
55
56User-agent: Cloudflare-Radar
57Disallow: /
58
59User-agent: CCBot
60Disallow: /
61
62User-agent: DeepSeekBot
63Disallow: /
64
65User-agent: DuckAssistBot
66Disallow: /
67
68User-agent: archive.org_bot
69Disallow: /
70
71User-agent: ia_archiver
72Disallow: /
73
74User-agent: MistralAI-User
75Disallow: /
76
77User-agent: Moreover
78Disallow: /
79
80User-agent: Adbeat
81Disallow: /
82
83User-agent: Argus
84Disallow: /
85
86User-agent: Baiduspider
87Disallow: /
88
89User-agent: Baiduspider-image
90Disallow: /
91
92User-agent: Bitdefender
93Disallow: /
94
95User-agent: Blogtrottr
96Disallow: /
97
98User-agent: BloombergBot
99Disallow: /
100
101User-agent: BoardReader
102Disallow: /
103
104User-agent: checkmy.ws
105Disallow: /
106
107User-agent: Clickagy
108Disallow: /
109
110User-agent: CookieBot
111Disallow: /
112
113User-agent: cXensebot
114Disallow: /
115
116User-agent: DataObserver
117Disallow: /
118
119User-agent: DomainStatsBot
120Disallow: /
121
122User-agent: Embedly
123Disallow: /
124
125User-agent: Feedspotbot
126Disallow: /
127
128User-agent: FlipboardProxy
129Disallow: /
130
131User-agent: Getintent
132Disallow: /
133
134User-agent: Google-Apps-Script
135Disallow: /
136
137User-agent: GoogleDrive
138Disallow: /
139
140User-agent: Grapevine6
141Disallow: /
142
143User-agent: Gumgum
144Disallow: /
145
146User-agent: HaloBot
147Disallow: /
148
149User-agent: HubSpot Crawler
150Disallow: /
151
152User-agent: IbouBot
153Disallow: /
154
155User-agent: Inout
156Disallow: /
157
158User-agent: LinerBot
159Disallow: /
160
161User-agent: MJ12bot
162Disallow: /
163
164User-agent: Manus
165Disallow: /
166
167User-agent: Meltwater
168Disallow: /
169
170User-agent: Metajob
171Disallow: /
172
173User-agent: Muckrack
174Disallow: /
175
176User-agent: NetSeer
177Disallow: /
178
179User-agent: Netvibes
180Disallow: /
181
182User-agent: Onyphe
183Disallow: /
184
185User-agent: OpenClaw
186Disallow: /
187
188User-agent: Opoint
189Disallow: /
190
191User-agent: OutbrainBot
192Disallow: /
193
194User-agent: PlumAnalytics
195Disallow: /
196
197User-agent: ProSitemaps
198Disallow: /
199
200User-agent: PerplexityBot
201Disallow: /
202
203User-agent: Perplexity-User
204Disallow: /
205
206User-agent: proximic
207Disallow: /
208
209User-agent: PubMatic
210Disallow: /
211
212User-agent: Quantcastbot
213Disallow: /
214
215User-agent: SearchConcepts
216Disallow: /
217
218User-agent: Semantic-Visions
219Disallow: /
220
221User-agent: SentiBot
222Disallow: /
223
224User-agent: Sindup
225Disallow: /
226
227User-agent: Sirdata
228Disallow: /
229
230User-agent: SmartologyBot
231Disallow: /
232
233User-agent: Superfeedr
234Disallow: /
235
236User-agent: surlybot
237Disallow: /
238
239User-agent: Synaptive
240Disallow: /
241
242User-agent: TinEye
243Disallow: /
244
245User-agent: Trendiction
246Disallow: /
247
248User-agent: TurnitinBot
249Disallow: /
250
251User-agent: Ubermetrics
252Disallow: /
253
254User-agent: Vebidoo
255Disallow: /
256
257User-agent: Webcorp
258Disallow: /
259
260User-agent: Weborama
261Disallow: /
262
263User-agent: WebPageTest
264Disallow: /
265
266User-agent: YandexAccessibilityBot
267Disallow: /
268
269User-agent: YandexDirect
270Disallow: /
271
272User-agent: YandexDirectDyn
273Disallow: /
274
275User-agent: YaDirectFetcher
276Disallow: /
277
278User-agent: YandexFavicons
279Disallow: /
280
281User-agent: YandexMetrika
282Disallow: /
283
284User-agent: YandexMobileBot
285Disallow: /
286
287User-agent: YandexNews
288Disallow: /
289
290User-agent: YandexRCA
291Disallow: /
292
293User-agent: YandexRenderResourcesBot
294Disallow: /
295
296User-agent: YandexUserproxy
297Disallow: /
298
299# OAI-AdsBot se mantiene permitido y no se bloquea.
300
301User-agent: *
302Disallow: /pruebas/
303Disallow: /publicidad/
304Disallow: /notificarelacionadas
305Disallow: /*.swf$
306Disallow: /eskupTSUpdate
307Disallow: /ThreadeskupSimple
308Disallow: /OuteskupSimple
309# Anadidos 20250325 por oscarf
310Disallow: /Ineskup
311Disallow: /Outeskup
312Disallow: /OuteskupSimpleMustache
313Disallow: /Msgeskup
314Disallow: /Profileeskup
315Disallow: /Threadeskup
316
317# Disallow: /.well-known/amphtml/
318Disallow: /t/
319Disallow: *?preview=true$
320
321# Bots bloqueados
322User-agent: 008
323Disallow: /
324User-agent: AddThis.com
325Disallow: /
326User-agent: admantx
327Disallow: /
328User-agent: AhrefsBot
329Disallow: /
330User-agent: BDCbot
331Disallow: /
332User-agent: Bender
333Disallow: /
334User-agent: BIXOCRAWLER
335Disallow: /
336User-agent: bl.uk_lddc_bot
337Disallow: /
338User-agent: BLEXBot
339Disallow: /
340User-agent: BUbiNG
341Disallow: /
342User-agent: Cliqzbot
343Disallow: /
344User-agent: CNCDialer
345Disallow: /
346User-agent: crawler4j
347Disallow: /
348User-agent: CrystalSemanticsBot
349Disallow: /
350User-agent: CyberAlert
351Disallow: /
352User-agent: DigExt
353Disallow: /
354User-agent: discobot
355Disallow: /
356User-agent: discoverybot
357Disallow: /
358User-agent: dloader
359Disallow: /
360User-agent: dloader(NaverRobot)
361Disallow: /
362User-agent: DOC
363Disallow: /
364User-agent: Download Ninja
365Disallow: /
366User-agent: DTS Agent
367Disallow: /
368User-agent: Exabot
369Disallow: /
370User-agent: Ezooms
371Disallow: /
372User-agent: FairShare
373Disallow: /
374User-agent: Fetch
375Disallow: /
376User-agent: Flamingo_SearchEngine
377Disallow: /
378User-agent: Genieo
379Disallow: /
380User-agent: Gigabot
381Disallow: /
382User-agent: grub-client
383Disallow: /
384User-agent: Heritrix
385Disallow: /
386User-agent: heritrix/3.3.0
387Disallow: /
388User-agent: HTTrack
389Disallow: /
390User-agent: integromedb
391Disallow: /
392User-agent: IstellaBot
393Disallow: /
394User-agent: JikeSpider
395Disallow: /
396User-agent: Jyxobot
397Disallow: /
398User-agent: k2spider
399Disallow: /
400User-agent: Kimengi
401Disallow: /
402User-agent: Kimengi/nineconnections.com
403Disallow: /
404User-agent: larbin
405Disallow: /
406User-agent: LexxeBot/1.0
407Disallow: /
408User-agent: libwww
409Disallow: /
410User-agent: linko
411Disallow: /
412User-agent: Livelapbot
413Disallow: /
414User-agent: magpie-crawler
415Disallow: /
416User-agent: Maxthon
417Disallow: /
418User-agent: MetaURI
419Disallow: /
420User-agent: Microsoft.URL.Control
421Disallow: /
422User-agent: Moreoverbot
423Disallow: /
424User-agent: MSIECrawler
425Disallow: /
426User-agent: nabot
427Disallow: /
428User-agent: NaverBot
429Disallow: /
430User-agent: NerdByNature.Bot
431Disallow: /
432User-agent: netEstate NE Crawler
433Disallow: /
434User-agent: NetSeer crawler
435Disallow: /
436User-agent: Newscan
437Disallow: /
438User-agent: NextGenSearchBot
439Disallow: /
440User-agent: NPBot
441Disallow: /
442User-agent: nutch
443Disallow: /
444User-agent: Offline Explorer
445Disallow: /
446User-agent: omgilibot
447Disallow: /
448User-agent: Orthogaffe
449Disallow: /
450User-agent: PiplBot
451Disallow: /
452User-agent: Pixray-Seeker
453Disallow: /
454User-agent: psbot
455Disallow: /
456User-agent: QuerySeekerSpider
457Disallow: /
458User-agent: seokicks
459Disallow: /
460User-agent: SEOkicks-Robot
461Disallow: /
462User-agent: SiteBot
463Disallow: /
464User-agent: SiteBot/0.1
465Disallow: /
466User-agent: sitecheck.internetseer.com
467Disallow: /
468User-agent: SiteSnagger
469Disallow: /
470User-agent: Slurp
471Disallow: /
472User-agent: sogou
473Disallow: /
474User-agent: Sosospider
475Disallow: /
476User-agent: Spinn3r
477Disallow: /
478User-agent: Teleport
479Disallow: /
480User-agent: TeleportPro
481Disallow: /
482User-agent: trendictionbot
483Disallow: /
484User-agent: trovitBot
485Disallow: /
486User-agent: UbiCrawler
487Disallow: /
488User-agent: uMBot-LN
489Disallow: /
490User-agent: UnisterBot
491Disallow: /
492User-agent: UniversalFeedParser
493Disallow: /
494User-agent: WBSearchBot
495Disallow: /
496User-agent: WebCopier
497Disallow: /
498User-agent: WebReaper
499Disallow: /
500User-agent: WebStripper
501Disallow: /
502User-agent: WebZIP
503Disallow: /
504User-agent: WeSEE:Search
505Disallow: /
506User-agent: wget
507Disallow: /
508User-agent: Wotbot
509Disallow: /
510User-agent: wotbox
511Disallow: /
512User-agent: Xenu
513Disallow: /
514User-agent: Yasni
515Disallow: /
516User-agent: Zao
517Disallow: /
518User-agent: Zealbot
519Disallow: /
520User-agent: ZyBORG
521Disallow: /