NeuralCrawl

El País / robots.txt snapshot

← back to elpais.com · fetched 2026-09-17T21:12:52Z (1d ago) · HTTP 200 · 7738 bytes · sha256 24c2d0a38586c6ae · raw

final URL: https://elpais.com/robots.txt

1User-agent: PetalBot
2Disallow: /
3
4# Bloque bots global confirmado 20260505
5User-agent: bedrock-agentcore-browser
6Disallow: /
7
8User-agent: Amazon-Bot
9Disallow: /
10
11User-agent: Amazonbot
12Disallow: /
13
14User-agent: AmazonProductDiscoveryBot
15Disallow: /
16
17User-agent: Amzn-SearchBot
18Disallow: /
19
20User-agent: Amzn-User
21Disallow: /
22
23User-agent: Arquivo-web-crawler
24Disallow: /
25
26User-agent: bnf.fr_bot
27Disallow: /
28
29User-agent: Bytespider
30Disallow: /
31
32User-agent: Claude-Extension
33Disallow: /
34
35User-agent: ClaudeBot
36Disallow: /
37
38User-agent: Claude-User
39Disallow: /
40
41User-agent: Claude-SearchBot
42Disallow: /
43
44User-agent: Comet
45Disallow: /
46
47User-agent: CloudFlare-AlwaysOnline
48Disallow: /
49
50User-agent: Cloudflare-Healthchecks
51Disallow: /
52
53User-agent: CloudflareBrowserRendering
54Disallow: /
55
56User-agent: Cloudflare-Radar
57Disallow: /
58
59User-agent: CCBot
60Disallow: /
61
62User-agent: DeepSeekBot
63Disallow: /
64
65User-agent: DuckAssistBot
66Disallow: /
67
68User-agent: archive.org_bot
69Disallow: /
70
71User-agent: ia_archiver
72Disallow: /
73
74User-agent: MistralAI-User
75Disallow: /
76
77User-agent: Moreover
78Disallow: /
79
80User-agent: Adbeat
81Disallow: /
82
83User-agent: Argus
84Disallow: /
85
86User-agent: Baiduspider
87Disallow: /
88
89User-agent: Baiduspider-image
90Disallow: /
91
92User-agent: Bitdefender
93Disallow: /
94
95User-agent: Blogtrottr
96Disallow: /
97
98User-agent: BloombergBot
99Disallow: /
100
101User-agent: BoardReader
102Disallow: /
103
104User-agent: checkmy.ws
105Disallow: /
106
107User-agent: Clickagy
108Disallow: /
109
110User-agent: CookieBot
111Disallow: /
112
113User-agent: cXensebot
114Disallow: /
115
116User-agent: DataObserver
117Disallow: /
118
119User-agent: DomainStatsBot
120Disallow: /
121
122User-agent: Embedly
123Disallow: /
124
125User-agent: Feedspotbot
126Disallow: /
127
128User-agent: FlipboardProxy
129Disallow: /
130
131User-agent: Getintent
132Disallow: /
133
134User-agent: Google-Apps-Script
135Disallow: /
136
137User-agent: GoogleDrive
138Disallow: /
139
140User-agent: Grapevine6
141Disallow: /
142
143User-agent: Gumgum
144Disallow: /
145
146User-agent: HaloBot
147Disallow: /
148
149User-agent: HubSpot Crawler
150Disallow: /
151
152User-agent: IbouBot
153Disallow: /
154
155User-agent: Inout
156Disallow: /
157
158User-agent: LinerBot
159Disallow: /
160
161User-agent: MJ12bot
162Disallow: /
163
164User-agent: Manus
165Disallow: /
166
167User-agent: Meltwater
168Disallow: /
169
170User-agent: Metajob
171Disallow: /
172
173User-agent: Muckrack
174Disallow: /
175
176User-agent: NetSeer
177Disallow: /
178
179User-agent: Netvibes
180Disallow: /
181
182User-agent: Onyphe
183Disallow: /
184
185User-agent: OpenClaw
186Disallow: /
187
188User-agent: Opoint
189Disallow: /
190
191User-agent: OutbrainBot
192Disallow: /
193
194User-agent: PlumAnalytics
195Disallow: /
196
197User-agent: ProSitemaps
198Disallow: /
199
200User-agent: PerplexityBot
201Disallow: /
202
203User-agent: Perplexity-User
204Disallow: /
205
206User-agent: proximic
207Disallow: /
208
209User-agent: PubMatic
210Disallow: /
211
212User-agent: Quantcastbot
213Disallow: /
214
215User-agent: SearchConcepts
216Disallow: /
217
218User-agent: Semantic-Visions
219Disallow: /
220
221User-agent: SentiBot
222Disallow: /
223
224User-agent: Sindup
225Disallow: /
226
227User-agent: Sirdata
228Disallow: /
229
230User-agent: SmartologyBot
231Disallow: /
232
233User-agent: Superfeedr
234Disallow: /
235
236User-agent: surlybot
237Disallow: /
238
239User-agent: Synaptive
240Disallow: /
241
242User-agent: TinEye
243Disallow: /
244
245User-agent: Trendiction
246Disallow: /
247
248User-agent: TurnitinBot
249Disallow: /
250
251User-agent: Ubermetrics
252Disallow: /
253
254User-agent: Vebidoo
255Disallow: /
256
257User-agent: Webcorp
258Disallow: /
259
260User-agent: Weborama
261Disallow: /
262
263User-agent: WebPageTest
264Disallow: /
265
266User-agent: YandexAccessibilityBot
267Disallow: /
268
269User-agent: YandexDirect
270Disallow: /
271
272User-agent: YandexDirectDyn
273Disallow: /
274
275User-agent: YaDirectFetcher
276Disallow: /
277
278User-agent: YandexFavicons
279Disallow: /
280
281User-agent: YandexMetrika
282Disallow: /
283
284User-agent: YandexMobileBot
285Disallow: /
286
287User-agent: YandexNews
288Disallow: /
289
290User-agent: YandexRCA
291Disallow: /
292
293User-agent: YandexRenderResourcesBot
294Disallow: /
295
296User-agent: YandexUserproxy
297Disallow: /
298
299# OAI-AdsBot se mantiene permitido y no se bloquea.
300
301User-agent: *
302Disallow: /pruebas/
303Disallow: /publicidad/
304Disallow: /notificarelacionadas
305Disallow: /*.swf$
306Disallow: /eskupTSUpdate
307Disallow: /ThreadeskupSimple
308Disallow: /OuteskupSimple
309# Anadidos 20250325 por oscarf
310Disallow: /Ineskup
311Disallow: /Outeskup
312Disallow: /OuteskupSimpleMustache
313Disallow: /Msgeskup
314Disallow: /Profileeskup
315Disallow: /Threadeskup
316Disallow: /*?*core_dmena=
317
318# Disallow: /.well-known/amphtml/
319Disallow: /t/
320Disallow: *?preview=true$
321
322# Bots bloqueados
323User-agent: 008
324Disallow: /
325User-agent: AddThis.com
326Disallow: /
327User-agent: admantx
328Disallow: /
329User-agent: AhrefsBot
330Disallow: /
331User-agent: BDCbot
332Disallow: /
333User-agent: Bender
334Disallow: /
335User-agent: BIXOCRAWLER
336Disallow: /
337User-agent: bl.uk_lddc_bot
338Disallow: /
339User-agent: BLEXBot
340Disallow: /
341User-agent: BUbiNG
342Disallow: /
343User-agent: Cliqzbot
344Disallow: /
345User-agent: CNCDialer
346Disallow: /
347User-agent: crawler4j
348Disallow: /
349User-agent: CrystalSemanticsBot
350Disallow: /
351User-agent: CyberAlert
352Disallow: /
353User-agent: DigExt
354Disallow: /
355User-agent: discobot
356Disallow: /
357User-agent: discoverybot
358Disallow: /
359User-agent: dloader
360Disallow: /
361User-agent: dloader(NaverRobot)
362Disallow: /
363User-agent: DOC
364Disallow: /
365User-agent: Download Ninja
366Disallow: /
367User-agent: DTS Agent
368Disallow: /
369User-agent: Exabot
370Disallow: /
371User-agent: Ezooms
372Disallow: /
373User-agent: FairShare
374Disallow: /
375User-agent: Fetch
376Disallow: /
377User-agent: Flamingo_SearchEngine
378Disallow: /
379User-agent: Genieo
380Disallow: /
381User-agent: Gigabot
382Disallow: /
383User-agent: grub-client
384Disallow: /
385User-agent: Heritrix
386Disallow: /
387User-agent: heritrix/3.3.0
388Disallow: /
389User-agent: HTTrack
390Disallow: /
391User-agent: integromedb
392Disallow: /
393User-agent: IstellaBot
394Disallow: /
395User-agent: JikeSpider
396Disallow: /
397User-agent: Jyxobot
398Disallow: /
399User-agent: k2spider
400Disallow: /
401User-agent: Kimengi
402Disallow: /
403User-agent: Kimengi/nineconnections.com
404Disallow: /
405User-agent: larbin
406Disallow: /
407User-agent: LexxeBot/1.0
408Disallow: /
409User-agent: libwww
410Disallow: /
411User-agent: linko
412Disallow: /
413User-agent: Livelapbot
414Disallow: /
415User-agent: magpie-crawler
416Disallow: /
417User-agent: Maxthon
418Disallow: /
419User-agent: MetaURI
420Disallow: /
421User-agent: Microsoft.URL.Control
422Disallow: /
423User-agent: Moreoverbot
424Disallow: /
425User-agent: MSIECrawler
426Disallow: /
427User-agent: nabot
428Disallow: /
429User-agent: NaverBot
430Disallow: /
431User-agent: NerdByNature.Bot
432Disallow: /
433User-agent: netEstate NE Crawler
434Disallow: /
435User-agent: NetSeer crawler
436Disallow: /
437User-agent: Newscan
438Disallow: /
439User-agent: NextGenSearchBot
440Disallow: /
441User-agent: NPBot
442Disallow: /
443User-agent: nutch
444Disallow: /
445User-agent: Offline Explorer
446Disallow: /
447User-agent: omgilibot
448Disallow: /
449User-agent: Orthogaffe
450Disallow: /
451User-agent: PiplBot
452Disallow: /
453User-agent: Pixray-Seeker
454Disallow: /
455User-agent: psbot
456Disallow: /
457User-agent: QuerySeekerSpider
458Disallow: /
459User-agent: seokicks
460Disallow: /
461User-agent: SEOkicks-Robot
462Disallow: /
463User-agent: SiteBot
464Disallow: /
465User-agent: SiteBot/0.1
466Disallow: /
467User-agent: sitecheck.internetseer.com
468Disallow: /
469User-agent: SiteSnagger
470Disallow: /
471User-agent: Slurp
472Disallow: /
473User-agent: sogou
474Disallow: /
475User-agent: Sosospider
476Disallow: /
477User-agent: Spinn3r
478Disallow: /
479User-agent: Teleport
480Disallow: /
481User-agent: TeleportPro
482Disallow: /
483User-agent: trendictionbot
484Disallow: /
485User-agent: trovitBot
486Disallow: /
487User-agent: UbiCrawler
488Disallow: /
489User-agent: uMBot-LN
490Disallow: /
491User-agent: UnisterBot
492Disallow: /
493User-agent: UniversalFeedParser
494Disallow: /
495User-agent: WBSearchBot
496Disallow: /
497User-agent: WebCopier
498Disallow: /
499User-agent: WebReaper
500Disallow: /
501User-agent: WebStripper
502Disallow: /
503User-agent: WebZIP
504Disallow: /
505User-agent: WeSEE:Search
506Disallow: /
507User-agent: wget
508Disallow: /
509User-agent: Wotbot
510Disallow: /
511User-agent: wotbox
512Disallow: /
513User-agent: Xenu
514Disallow: /
515User-agent: Yasni
516Disallow: /
517User-agent: Zao
518Disallow: /
519User-agent: Zealbot
520Disallow: /
521User-agent: ZyBORG
522Disallow: /