NeuralCrawl

El País / robots.txt snapshot

← back to elpais.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 7506 bytes · sha256 350cd40560747ecc · raw

final URL: https://elpais.com/robots.txt

1User-agent: PetalBot
2Disallow: /
3
4# Bloque bots global confirmado 20260505
5User-agent: bedrock-agentcore-browser
6Disallow: /
7
8User-agent: Amazon-Bot
9Disallow: /
10
11User-agent: Amazonbot
12Disallow: /
13
14User-agent: AmazonProductDiscoveryBot
15Disallow: /
16
17User-agent: Amzn-SearchBot
18Disallow: /
19
20User-agent: Amzn-User
21Disallow: /
22
23User-agent: Arquivo-web-crawler
24Disallow: /
25
26User-agent: bnf.fr_bot
27Disallow: /
28
29User-agent: Bytespider
30Disallow: /
31
32User-agent: Claude-Extension
33Disallow: /
34
35User-agent: ClaudeBot
36Disallow: /
37
38User-agent: Claude-User
39Disallow: /
40
41User-agent: CloudFlare-AlwaysOnline
42Disallow: /
43
44User-agent: Cloudflare-Healthchecks
45Disallow: /
46
47User-agent: CloudflareBrowserRendering
48Disallow: /
49
50User-agent: Cloudflare-Radar
51Disallow: /
52
53User-agent: CCBot
54Disallow: /
55
56User-agent: DeepSeekBot
57Disallow: /
58
59User-agent: DuckAssistBot
60Disallow: /
61
62User-agent: archive.org_bot
63Disallow: /
64
65User-agent: ia_archiver
66Disallow: /
67
68User-agent: MistralAI-User
69Disallow: /
70
71User-agent: Moreover
72Disallow: /
73
74User-agent: Adbeat
75Disallow: /
76
77User-agent: Argus
78Disallow: /
79
80User-agent: Baiduspider
81Disallow: /
82
83User-agent: Baiduspider-image
84Disallow: /
85
86User-agent: Bitdefender
87Disallow: /
88
89User-agent: Blogtrottr
90Disallow: /
91
92User-agent: BloombergBot
93Disallow: /
94
95User-agent: BoardReader
96Disallow: /
97
98User-agent: checkmy.ws
99Disallow: /
100
101User-agent: Clickagy
102Disallow: /
103
104User-agent: CookieBot
105Disallow: /
106
107User-agent: cXensebot
108Disallow: /
109
110User-agent: DataObserver
111Disallow: /
112
113User-agent: DomainStatsBot
114Disallow: /
115
116User-agent: Embedly
117Disallow: /
118
119User-agent: Feedspotbot
120Disallow: /
121
122User-agent: FlipboardProxy
123Disallow: /
124
125User-agent: Getintent
126Disallow: /
127
128User-agent: Google-Apps-Script
129Disallow: /
130
131User-agent: GoogleDrive
132Disallow: /
133
134User-agent: Grapevine6
135Disallow: /
136
137User-agent: Gumgum
138Disallow: /
139
140User-agent: HaloBot
141Disallow: /
142
143User-agent: HubSpot Crawler
144Disallow: /
145
146User-agent: IbouBot
147Disallow: /
148
149User-agent: Inout
150Disallow: /
151
152User-agent: LinerBot
153Disallow: /
154
155User-agent: MJ12bot
156Disallow: /
157
158User-agent: Manus
159Disallow: /
160
161User-agent: Meltwater
162Disallow: /
163
164User-agent: Metajob
165Disallow: /
166
167User-agent: Muckrack
168Disallow: /
169
170User-agent: NetSeer
171Disallow: /
172
173User-agent: Netvibes
174Disallow: /
175
176User-agent: Onyphe
177Disallow: /
178
179User-agent: OpenClaw
180Disallow: /
181
182User-agent: Opoint
183Disallow: /
184
185User-agent: OutbrainBot
186Disallow: /
187
188User-agent: PlumAnalytics
189Disallow: /
190
191User-agent: ProSitemaps
192Disallow: /
193
194User-agent: proximic
195Disallow: /
196
197User-agent: PubMatic
198Disallow: /
199
200User-agent: Quantcastbot
201Disallow: /
202
203User-agent: SearchConcepts
204Disallow: /
205
206User-agent: Semantic-Visions
207Disallow: /
208
209User-agent: SentiBot
210Disallow: /
211
212User-agent: Sindup
213Disallow: /
214
215User-agent: Sirdata
216Disallow: /
217
218User-agent: SmartologyBot
219Disallow: /
220
221User-agent: Superfeedr
222Disallow: /
223
224User-agent: surlybot
225Disallow: /
226
227User-agent: Synaptive
228Disallow: /
229
230User-agent: TinEye
231Disallow: /
232
233User-agent: Trendiction
234Disallow: /
235
236User-agent: TurnitinBot
237Disallow: /
238
239User-agent: Ubermetrics
240Disallow: /
241
242User-agent: Vebidoo
243Disallow: /
244
245User-agent: Webcorp
246Disallow: /
247
248User-agent: Weborama
249Disallow: /
250
251User-agent: WebPageTest
252Disallow: /
253
254User-agent: YandexAccessibilityBot
255Disallow: /
256
257User-agent: YandexDirect
258Disallow: /
259
260User-agent: YandexDirectDyn
261Disallow: /
262
263User-agent: YaDirectFetcher
264Disallow: /
265
266User-agent: YandexFavicons
267Disallow: /
268
269User-agent: YandexMetrika
270Disallow: /
271
272User-agent: YandexMobileBot
273Disallow: /
274
275User-agent: YandexNews
276Disallow: /
277
278User-agent: YandexRCA
279Disallow: /
280
281User-agent: YandexRenderResourcesBot
282Disallow: /
283
284User-agent: YandexUserproxy
285Disallow: /
286
287User-agent: *
288Disallow: /pruebas/
289Disallow: /publicidad/
290Disallow: /notificarelacionadas
291Disallow: /*.swf$
292Disallow: /eskupTSUpdate
293Disallow: /ThreadeskupSimple
294Disallow: /OuteskupSimple
295# Anadidos 20250325 por oscarf
296Disallow: /Ineskup
297Disallow: /Outeskup
298Disallow: /OuteskupSimpleMustache
299Disallow: /Msgeskup
300Disallow: /Profileeskup
301Disallow: /Threadeskup
302
303# Disallow: /.well-known/amphtml/
304Disallow: /t/
305Disallow: *?preview=true$
306
307# Bots bloqueados
308User-agent: 008
309Disallow: /
310User-agent: AddThis.com
311Disallow: /
312User-agent: admantx
313Disallow: /
314User-agent: AhrefsBot
315Disallow: /
316User-agent: BDCbot
317Disallow: /
318User-agent: Bender
319Disallow: /
320User-agent: BIXOCRAWLER
321Disallow: /
322User-agent: bl.uk_lddc_bot
323Disallow: /
324User-agent: BLEXBot
325Disallow: /
326User-agent: BUbiNG
327Disallow: /
328User-agent: Cliqzbot
329Disallow: /
330User-agent: CNCDialer
331Disallow: /
332User-agent: crawler4j
333Disallow: /
334User-agent: CrystalSemanticsBot
335Disallow: /
336User-agent: CyberAlert
337Disallow: /
338User-agent: DigExt
339Disallow: /
340User-agent: discobot
341Disallow: /
342User-agent: discoverybot
343Disallow: /
344User-agent: dloader
345Disallow: /
346User-agent: dloader(NaverRobot)
347Disallow: /
348User-agent: DOC
349Disallow: /
350User-agent: Download Ninja
351Disallow: /
352User-agent: DTS Agent
353Disallow: /
354User-agent: Exabot
355Disallow: /
356User-agent: Ezooms
357Disallow: /
358User-agent: FairShare
359Disallow: /
360User-agent: Fetch
361Disallow: /
362User-agent: Flamingo_SearchEngine
363Disallow: /
364User-agent: Genieo
365Disallow: /
366User-agent: Gigabot
367Disallow: /
368User-agent: grub-client
369Disallow: /
370User-agent: Heritrix
371Disallow: /
372User-agent: heritrix/3.3.0
373Disallow: /
374User-agent: HTTrack
375Disallow: /
376User-agent: integromedb
377Disallow: /
378User-agent: IstellaBot
379Disallow: /
380User-agent: JikeSpider
381Disallow: /
382User-agent: Jyxobot
383Disallow: /
384User-agent: k2spider
385Disallow: /
386User-agent: Kimengi
387Disallow: /
388User-agent: Kimengi/nineconnections.com
389Disallow: /
390User-agent: larbin
391Disallow: /
392User-agent: LexxeBot/1.0
393Disallow: /
394User-agent: libwww
395Disallow: /
396User-agent: linko
397Disallow: /
398User-agent: Livelapbot
399Disallow: /
400User-agent: magpie-crawler
401Disallow: /
402User-agent: Maxthon
403Disallow: /
404User-agent: MetaURI
405Disallow: /
406User-agent: Microsoft.URL.Control
407Disallow: /
408User-agent: Moreoverbot
409Disallow: /
410User-agent: MSIECrawler
411Disallow: /
412User-agent: nabot
413Disallow: /
414User-agent: NaverBot
415Disallow: /
416User-agent: NerdByNature.Bot
417Disallow: /
418User-agent: netEstate NE Crawler
419Disallow: /
420User-agent: NetSeer crawler
421Disallow: /
422User-agent: Newscan
423Disallow: /
424User-agent: NextGenSearchBot
425Disallow: /
426User-agent: NPBot
427Disallow: /
428User-agent: nutch
429Disallow: /
430User-agent: Offline Explorer
431Disallow: /
432User-agent: omgilibot
433Disallow: /
434User-agent: Orthogaffe
435Disallow: /
436User-agent: PiplBot
437Disallow: /
438User-agent: Pixray-Seeker
439Disallow: /
440User-agent: psbot
441Disallow: /
442User-agent: QuerySeekerSpider
443Disallow: /
444User-agent: seokicks
445Disallow: /
446User-agent: SEOkicks-Robot
447Disallow: /
448User-agent: SiteBot
449Disallow: /
450User-agent: SiteBot/0.1
451Disallow: /
452User-agent: sitecheck.internetseer.com
453Disallow: /
454User-agent: SiteSnagger
455Disallow: /
456User-agent: Slurp
457Disallow: /
458User-agent: sogou
459Disallow: /
460User-agent: Sosospider
461Disallow: /
462User-agent: Spinn3r
463Disallow: /
464User-agent: Teleport
465Disallow: /
466User-agent: TeleportPro
467Disallow: /
468User-agent: trendictionbot
469Disallow: /
470User-agent: trovitBot
471Disallow: /
472User-agent: UbiCrawler
473Disallow: /
474User-agent: uMBot-LN
475Disallow: /
476User-agent: UnisterBot
477Disallow: /
478User-agent: UniversalFeedParser
479Disallow: /
480User-agent: WBSearchBot
481Disallow: /
482User-agent: WebCopier
483Disallow: /
484User-agent: WebReaper
485Disallow: /
486User-agent: WebStripper
487Disallow: /
488User-agent: WebZIP
489Disallow: /
490User-agent: WeSEE:Search
491Disallow: /
492User-agent: wget
493Disallow: /
494User-agent: Wotbot
495Disallow: /
496User-agent: wotbox
497Disallow: /
498User-agent: Xenu
499Disallow: /
500User-agent: Yasni
501Disallow: /
502User-agent: Zao
503Disallow: /
504User-agent: Zealbot
505Disallow: /
506User-agent: ZyBORG
507Disallow: /