NeuralCrawl

Le Figaro / robots.txt snapshot

← back to lefigaro.fr · fetched 2026-07-27T15:48:23Z (9d ago) · HTTP 200 · 12492 bytes · sha256 140b4d9d97654346 · raw

final URL: https://www.lefigaro.fr/robots.txt

1# :.
2# ++.
3# .+**:
4# .+****.
5# =******..
6# .********:.
7# -*********-.
8# =***********#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
9# .-************#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
10# .++=*************#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
11# .+****************FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
12# -****************#FFFFFFFF# .#FFFFFFFF#.
13# .+***************#FFFFFFF# .=+++++++++.
14# .+**************#FFFFFF#
15# .-+************FFFFFF#
16# .-**************FFFFFFFFFFFFFFFFF:
17# .-************FFFFFFFFFFFFFFFFF:
18# ..=*********FFFFFFFFFFFFFFFFF:
19# .##******FFFFFFFFFFFFFFFFF:
20# #FFFF##*FFFFFF#*+.
21# #FFFFFFFFFFFFF#***=.
22# #FFFFFFFFFFFFF# .:+*-.
23# .#FFFFFFFFFFFFF#. -*-
24# +FFFFFFFFFFFFFFFFFFFFFFF- :+.
25# +FFFFFFFFFFFFFFFFFFFFFFF- .-=.
26# +FFFFFFFFFFFFFFFFFFFFFFF- .=-.
27# +FFFFFFFFFFFFFFFFFFFFFFF- .+.
28
29# Toute utilisation de nos contenus protégés autre qu'un usage strictement individuel (incluant notamment l'entraînement des grands modèles de langage (LLM),
30# l'entraînement des outils d'intelligence artificielle, la veille web ou le media monitoring) est conditionnée à la conclusion d'une licence d'utilisation
31# avec la SOCIETE DU FIGARO. Nous vous invitons à cette fin à contacter [email protected].
32#
33# Toute utilisation non-autorisée de nos contenus protégés est constitutive d'une contrefaçon de droit d'auteur et/ou du droit de producteur
34# de base de données et susceptible d'être poursuivie.
35#
36# Il est interdit de crawler notre site web en utilisant un agent d'utilisateur (user agent) volé qui ne correspond pas à votre identité.
37# L'utilisation des robots d'indexation web ou d'autres méthodes automatiques de feuilletage ou de navigation sur ce site Web n'est pas autorisée.
38
39#Généralités
40
41User-agent: *
42Disallow: /async/
43Disallow: /*?noheader*
44Disallow: /*?*&noheader*
45Disallow: /*?sfdebug*
46Disallow: /*?*&sfdebug*
47Disallow: /brouillon/
48Disallow: /proxy/
49Disallow: /synsearch/
50Disallow: /a-savoir-en-france/
51Disallow: /vos-questions
52
53#LLM
54
55User-agent: ChatGPT-User
56Allow: /voyages
57Allow: /culture
58Allow: /style
59Allow: /beaute
60Allow: /bons-plans
61Allow: /elections/resultats
62Disallow: /
63
64User-agent: Claude-SearchBot
65Allow: /voyages
66Allow: /bons-plans
67Allow: /elections/resultats
68Disallow: /
69
70User-agent: Claude-Web
71Allow: /voyages
72Allow: /bons-plans
73Allow: /elections/resultats
74Disallow: /
75
76User-agent: OAI-SearchBot
77Allow: /voyages
78Allow: /culture
79Allow: /style
80Allow: /beaute
81Allow: /bons-plans
82Allow: /elections/resultats
83Disallow: /
84
85User-agent: Claude-User
86Allow: /voyages
87Allow: /bons-plans
88Allow: /elections/resultats
89Disallow: /
90
91User-agent: ChatGPT Agent
92Allow: /voyages
93Allow: /bons-plans
94Allow: /elections/resultats
95Disallow: /
96
97User-agent: GoogleAgent-Mariner
98Allow: /voyages
99Allow: /bons-plans
100Allow: /elections/resultats
101Disallow: /
102
103User-agent: Google-NotebookLM
104Allow: /voyages
105Allow: /bons-plans
106Allow: /elections/resultats
107Disallow: /
108
109User-agent: Google-CloudVertexBot
110Allow: /voyages
111Allow: /bons-plans
112Allow: /elections/resultats
113Disallow: /
114
115User-agent: MistralAI-User
116Allow: /voyages
117Allow: /bons-plans
118Allow: /elections/resultats
119Disallow: /
120
121User-agent: Gemini-Deep-Research
122Allow: /voyages
123Allow: /bons-plans
124Allow: /elections/resultats
125Disallow: /
126
127User-agent: CloudVertexBot
128Disallow: /
129
130User-agent: anthropic-ai
131Disallow: /
132
133User-agent: CCBot
134Disallow: /
135
136User-agent: ClaudeBot
137Disallow: /
138
139User-agent: cohere-ai
140Disallow: /
141
142User-agent: CohereBot
143Disallow: /
144
145User-agent: GPTBot
146Disallow: /
147
148User-agent: Google-Extended
149Disallow: /
150
151User-agent: Kimi-SearchBot
152Disallow: /
153
154User-agent: Kimi-User
155Disallow: /
156
157User-agent: KimiBot
158Disallow: /
159
160#Crawler
161
162User-agent: Alexibot
163Disallow: /
164
165User-agent: AlvinetSpider
166Disallow: /
167
168User-agent: AmiSoftware
169Disallow: /
170
171User-agent: Antenne Hatena
172Disallow: /
173
174User-agent: ApifyBot
175Disallow: /
176
177User-agent: ApifyWebsiteContentCrawler
178Disallow: /
179
180User-agent: ApocalXExplorerBot
181Disallow: /
182
183User-agent: Argus
184Disallow: /
185
186User-agent: Ask n read
187Disallow: /
188
189User-agent: asknread.com
190Disallow: /
191
192User-agent: asterias
193Disallow: /
194
195User-agent: BlowFish/1.0
196Disallow: /
197
198User-agent: BotALot
199Disallow: /
200
201User-agent: Brightbot
202Disallow: /
203
204User-agent: BuiltBotTough
205Disallow: /
206
207User-agent: Bullseye/1.0
208Disallow: /
209
210User-agent: BunnySlippers
211Disallow: /
212
213User-agent: Cegbfeieh
214Disallow: /
215
216User-agent: CheeseBot
217Disallow: /
218
219User-agent: ConveraCrawler
220Disallow: /
221
222User-agent: cosmos
223Disallow: /
224
225User-agent: CragCrawler
226Disallow: /
227
228User-agent: Crescent
229Disallow: /
230
231User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
232Disallow: /
233
234User-agent: cydralspider
235Disallow: /
236
237User-agent: Diffbot
238Disallow: /
239
240User-agent: Diffbot-User
241Disallow: /
242
243User-agent: DISCo Pump 3.1
244Disallow: /
245
246User-agent: DittoSpyder
247Disallow: /
248
249User-agent: EroCrawler
250Disallow: /
251
252User-agent: eureka
253Disallow: /
254
255User-agent: ExaBot
256Disallow: /
257
258User-agent: Explore
259Disallow: /
260
261User-agent: Fetch
262Disallow: /
263
264User-agent: FirecrawlAgent
265Disallow: /
266
267User-agent: Flamingo_SearchEngine
268Disallow: /
269
270User-agent: Foobot
271Disallow: /
272
273User-agent: gammaSpider
274Disallow: /
275
276User-agent: GeistHaus-PageFetcher
277Disallow: /
278
279User-agent: grub-client
280Disallow: /
281
282User-agent: hloader
283Disallow: /
284
285User-agent: httplib
286Disallow: /
287
288User-agent: humanlinks
289Disallow: /
290
291User-agent: ia_archiver
292Allow: /$
293Disallow: /
294
295User-agent: ia_archiver-web.archive.org
296Allow: /$
297Disallow: /
298
299User-agent: indexer
300Disallow: /
301
302User-agent: InfoNaviRobot
303Disallow: /
304
305User-agent: infoseek
306Disallow: /
307
308User-agent: JennyBot
309Disallow: /
310
311User-agent: Jetbot
312Disallow: /
313
314User-agent: JikeSpider
315Disallow: /
316
317User-agent: k2spider
318Disallow: /
319
320User-agent: Kenjin Spider
321Disallow: /
322
323User-agent: larbin
324Disallow: /
325
326User-agent: LexiBot
327Disallow: /
328
329User-agent: libWeb/clsHTTP
330Disallow: /
331
332User-agent: libwww
333Disallow: /
334
335User-agent: linko
336Disallow: /
337
338User-agent: LinkScan/8.1a Unix
339Disallow: /
340
341User-agent: LinkWalker
342Disallow: /
343
344User-agent: lwp-trivial
345Disallow: /
346
347User-agent: lwp-trivial/1.34
348Disallow: /
349
350User-agent: Mata Hari
351Disallow: /
352
353User-agent: Mister PiX
354Disallow: /
355
356User-agent: MLBot
357Disallow: /
358
359User-agent: moget
360Disallow: /
361
362User-agent: moget/2.1
363Disallow: /
364
365User-agent: Naverbot
366Disallow: /
367
368User-agent: NetAttache
369Disallow: /
370
371User-agent: Newzbin
372Disallow: /
373
374User-agent: NICErsPRO
375Disallow: /
376
377User-agent: NPBot
378Disallow: /
379
380User-agent: ObjectsSearch
381Disallow: /
382
383User-agent: omgili
384Disallow: /
385
386User-agent: omgilibot
387Disallow: /
388
389User-agent: Openfind
390Disallow: /
391
392User-agent: OpenindexSpider
393Disallow: /
394
395User-agent: Pimptrain
396Disallow: /
397
398User-agent: ProPowerBot/2.14
399Disallow: /
400
401User-agent: ProWebWalker
402Disallow: /
403
404User-agent: psbot
405Disallow: /
406
407User-agent: QuepasaCreep
408Disallow: /
409
410User-agent: Querit-SearchBot
411Disallow: /
412
413User-agent: QueritBot
414Disallow: /
415
416User-agent: QueryN Metasearch
417Disallow: /
418
419User-agent: Raven
420Disallow: /
421
422User-agent: RepoMonkey
423Disallow: /
424
425User-agent: RMA
426Disallow: /
427
428User-agent: Scrapy
429Disallow: /
430
431User-agent: ShapBot
432Disallow: /
433
434User-agent: SiteBot
435Disallow: /
436
437User-agent: Sogou web spider
438Disallow: /
439
440User-agent: sosospider
441Disallow: /
442
443User-agent: SpankBot
444Disallow: /
445
446User-agent: Speedy
447Disallow: /
448
449User-agent: spotter
450Disallow: /
451
452User-agent: suggybot
453Disallow: /
454
455User-agent: SuperBot
456Disallow: /
457
458User-agent: SuperBot/2.6
459Disallow: /
460
461User-agent: suzuran
462Disallow: /
463
464User-agent: Szukacz/1.4
465Disallow: /
466
467User-agent: Telesoft
468Disallow: /
469
470User-agent: The Intraformant
471Disallow: /
472
473User-agent: TheNomad
474Disallow: /
475
476User-agent: TightTwatBot
477Disallow: /
478
479User-agent: Titan
480Disallow: /
481
482User-agent: toCrawl/UrlDispatcher
483Disallow: /
484
485User-agent: TosCrawler
486Disallow: /
487
488User-agent: True_Robot
489Disallow: /
490
491User-agent: True_Robot/1.0
492Disallow: /
493
494User-agent: turingos
495Disallow: /
496
497User-agent: URLy Warning
498Disallow: /
499
500User-agent: VCI
501Disallow: /
502
503User-agent: vsw
504Disallow: /
505
506User-agent: wapspider
507Disallow: /
508
509User-agent: WebBandit
510Disallow: /
511
512User-agent: WebBandit/3.50
513Disallow: /
514
515User-agent: WebEnhancer
516Disallow: /
517
518User-agent: WebmasterWorldForumBot
519Disallow: /
520
521User-agent: Webster Pro
522Disallow: /
523
524User-agent: WebZinger
525Disallow: /
526
527User-agent: Webzio-Extended
528Disallow: /
529
530User-agent: WWW-Collector-E
531Disallow: /
532
533User-agent: yacy
534Disallow: /
535
536User-agent: yandex
537Disallow: /
538
539User-agent: YRSPider
540Disallow: /
541
542User-agent: Zealbot
543Disallow: /
544
545User-agent: Zeus
546Disallow: /
547
548User-agent: Zite
549Disallow: /
550
551User-agent: Zookabot
552Disallow: /
553
554User-agent: ZyBORG
555Disallow: /
556
557#Outil
558
559User-agent: 5emeRue
560Disallow: /
561
562User-agent: 5erue
563Disallow: /
564
565User-agent: adequat
566Disallow: /
567
568User-agent: adequat-systems
569Disallow: /
570
571User-agent: Amazonbot
572Disallow: /
573
574User-agent: Augure
575Disallow: /
576
577User-agent: auramundi
578Disallow: /
579
580User-agent: BizInformation
581Disallow: /
582
583User-agent: Bloomberg
584Disallow: /
585
586User-agent: CherryPicker
587Disallow: /
588
589User-agent: CherryPickerElite/1.0
590Disallow: /
591
592User-agent: CherryPickerSE/1.0
593Disallow: /
594
595User-agent: Cision
596Disallow: /
597
598User-agent: coexel
599Disallow: /
600
601User-agent: CopyRightCheck
602Disallow: /
603
604User-agent: Corporama
605Disallow: /
606
607User-agent: Digimind
608Disallow: /
609
610User-agent: dotbot
611Disallow: /
612
613User-agent: Download Ninja
614Disallow: /
615
616User-agent: downloadexpress
617Disallow: /
618
619User-agent: ellisphere
620Disallow: /
621
622User-agent: EmailCollector
623Disallow: /
624
625User-agent: EmailSiphon
626Disallow: /
627
628User-agent: EmailWolf
629Disallow: /
630
631User-agent: Europresse
632Disallow: /
633
634User-agent: ExtractorPro
635Disallow: /
636
637User-agent: Fasterfox
638Disallow: /
639
640User-agent: HTTrack
641Disallow: /
642
643User-agent: HTTrack 3.0
644Disallow: /
645
646User-agent: Igentia
647Disallow: /
648
649User-agent: Kantar
650Disallow: /
651
652User-agent: kbcrawl
653Disallow: /
654
655User-agent: Knowings
656Disallow: /
657
658User-agent: leadbox
659Disallow: /
660
661User-agent: LinkextractorPro
662Disallow: /
663
664User-agent: linkfluence
665Disallow: /
666
667User-agent: manageo
668Disallow: /
669
670User-agent: mediacompil
671Disallow: /
672
673User-agent: mention
674Disallow: /
675
676User-agent: Moreover
677Disallow: /
678
679User-agent: mytwip
680Disallow: /
681
682User-agent: NetAnts
683Disallow: /
684
685User-agent: NetMechanic
686Disallow: /
687
688User-agent: newscan-online
689Disallow: /
690
691User-agent: NewsNow
692Disallow: /
693
694User-agent: Offline Explorer
695Disallow: /
696
697User-agent: opinion-tracker
698Disallow: /
699
700User-agent: Qwam content intelligence
701Disallow: /
702
703User-agent: readability.com
704Disallow: /
705
706User-agent: scoop.it
707Disallow: /
708
709User-agent: score3
710Disallow: /
711
712User-agent: SemrushBot
713Disallow: /
714
715User-agent: SightupBot
716Disallow: /
717
718User-agent: Sindup
719Disallow: /
720
721User-agent: SirdataBot
722Disallow: /
723
724User-agent: SiteSnagger
725Disallow: /
726
727User-agent: SiteSucker
728Disallow: /
729
730User-agent: Synthesio
731Disallow: /
732
733User-agent: Talkwalker
734Disallow: /
735
736User-agent: TavilyBot
737Disallow: /
738
739User-agent: Teleport
740Disallow: /
741
742User-agent: TeleportPro
743Disallow: /
744
745User-agent: trendeo
746Disallow: /
747
748User-agent: trendybuzz
749Disallow: /
750
751User-agent: TurnitinBot
752Disallow: /
753
754User-agent: up2news
755Disallow: /
756
757User-agent: UrlPouls
758Disallow: /
759
760User-agent: UseAI
761Disallow: /
762
763User-agent: vecteurplus
764Disallow: /
765
766User-agent: Verif
767Disallow: /
768
769User-agent: verticalsearch
770Disallow: /
771
772User-agent: Web Image Collector
773Disallow: /
774
775User-agent: WebAuto
776Disallow: /
777
778User-agent: WebCopier
779Disallow: /
780
781User-agent: webcopy
782Disallow: /
783
784User-agent: Webedia
785Disallow: /
786
787User-agent: webmirror
788Disallow: /
789
790User-agent: WebReaper
791Disallow: /
792
793User-agent: WebSauger
794Disallow: /
795
796User-agent: website extractor
797Disallow: /
798
799User-agent: Website Quester
800Disallow: /
801
802User-agent: WebStripper
803Disallow: /
804
805User-agent: WebStripper/2.02
806Disallow: /
807
808User-agent: WebZIP
809Disallow: /
810
811User-agent: Wget
812Disallow: /
813
814User-agent: WikioFeedBot
815Disallow: /
816
817User-agent: winello
818Disallow: /
819
820User-agent: WinHTTrack
821Disallow: /
822
823User-agent: Xenu Link Sleuth/1.3.8
824Disallow: /
825
826User-agent: YouBot
827Disallow: /
828
829User-agent: Youmag
830Disallow: /
831
832Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/articles.xml
833Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/sections.xml
834Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/topics.xml
835Sitemap: https://www.lefigaro.fr/sitemap_news.xml
836Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/elections/resultats/sitemap.xml
837Sitemap: https://static.lefigaro.fr/f1/lefigaro/sitemaps/robots-service.txt