NeuralCrawl

Le Figaro / robots.txt snapshot

← back to lefigaro.fr · fetched 2026-07-08T02:36:30Z (29d ago) · HTTP 200 · 12498 bytes · sha256 700cbb92c694723f · raw

final URL: https://www.lefigaro.fr/robots.txt

1# :.
2# ++.
3# .+**:
4# .+****.
5# =******..
6# .********:.
7# -*********-.
8# =***********#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
9# .-************#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
10# .++=*************#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
11# .+****************FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
12# -****************#FFFFFFFF# .#FFFFFFFF#.
13# .+***************#FFFFFFF# .=+++++++++.
14# .+**************#FFFFFF#
15# .-+************FFFFFF#
16# .-**************FFFFFFFFFFFFFFFFF:
17# .-************FFFFFFFFFFFFFFFFF:
18# ..=*********FFFFFFFFFFFFFFFFF:
19# .##******FFFFFFFFFFFFFFFFF:
20# #FFFF##*FFFFFF#*+.
21# #FFFFFFFFFFFFF#***=.
22# #FFFFFFFFFFFFF# .:+*-.
23# .#FFFFFFFFFFFFF#. -*-
24# +FFFFFFFFFFFFFFFFFFFFFFF- :+.
25# +FFFFFFFFFFFFFFFFFFFFFFF- .-=.
26# +FFFFFFFFFFFFFFFFFFFFFFF- .=-.
27# +FFFFFFFFFFFFFFFFFFFFFFF- .+.
28
29# Toute utilisation de nos contenus protégés autre qu'un usage strictement individuel (incluant notamment l'entraînement des grands modèles de langage (LLM),
30# l'entraînement des outils d'intelligence artificielle, la veille web ou le media monitoring) est conditionnée à la conclusion d'une licence d'utilisation
31# avec la SOCIETE DU FIGARO. Nous vous invitons à cette fin à contacter [email protected].
32#
33# Toute utilisation non-autorisée de nos contenus protégés est constitutive d'une contrefaçon de droit d'auteur et/ou du droit de producteur
34# de base de données et susceptible d'être poursuivie.
35#
36# Il est interdit de crawler notre site web en utilisant un agent d'utilisateur (user agent) volé qui ne correspond pas à votre identité.
37# L'utilisation des robots d'indexation web ou d'autres méthodes automatiques de feuilletage ou de navigation sur ce site Web n'est pas autorisée.
38
39#Généralités
40
41User-agent: *
42Disallow: /async/
43Disallow: /*?noheader*
44Disallow: /*?*&noheader*
45Disallow: /*?sfdebug*
46Disallow: /*?*&sfdebug*
47Disallow: /brouillon/
48Disallow: /proxy/
49Disallow: /synsearch/
50Disallow: /a-savoir-en-france/
51Disallow: /vos-questions
52
53#LLM
54
55User-agent: ChatGPT-User
56Allow: /voyages
57Allow: /culture
58Allow: /style
59Allow: /bons-plans
60Allow: /elections/resultats
61Disallow: /
62
63User-agent: Claude-SearchBot
64Allow: /voyages
65Allow: /bons-plans
66Allow: /elections/resultats
67Disallow: /
68
69User-agent: Claude-Web
70Allow: /voyages
71Allow: /bons-plans
72Allow: /elections/resultats
73Disallow: /
74
75User-agent: OAI-SearchBot
76Allow: /voyages
77Allow: /culture
78Allow: /style
79Allow: /bons-plans
80Allow: /elections/resultats
81Disallow: /
82
83User-agent: Claude-User
84Allow: /voyages
85Allow: /bons-plans
86Allow: /elections/resultats
87Disallow: /
88
89User-agent: ChatGPT Agent
90Allow: /voyages
91Allow: /bons-plans
92Allow: /elections/resultats
93Disallow: /
94
95User-agent: GoogleAgent-Mariner
96Allow: /voyages
97Allow: /bons-plans
98Allow: /elections/resultats
99Disallow: /
100
101User-agent: Google-NotebookLM
102Allow: /voyages
103Allow: /bons-plans
104Allow: /elections/resultats
105Disallow: /
106
107User-agent: Google-CloudVertexBot
108Allow: /voyages
109Allow: /bons-plans
110Allow: /elections/resultats
111Disallow: /
112
113User-agent: MistralAI-User
114Allow: /voyages
115Allow: /bons-plans
116Allow: /elections/resultats
117Disallow: /
118
119User-agent: Gemini-Deep-Research
120Allow: /voyages
121Allow: /bons-plans
122Allow: /elections/resultats
123Disallow: /
124
125User-agent: CloudVertexBot
126Disallow: /
127
128User-agent: anthropic-ai
129Disallow: /
130
131User-agent: Bytespider
132Disallow: /
133
134User-agent: CCBot
135Disallow: /
136
137User-agent: ClaudeBot
138Disallow: /
139
140User-agent: cohere-ai
141Disallow: /
142
143User-agent: CohereBot
144Disallow: /
145
146User-agent: GPTBot
147Disallow: /
148
149User-agent: Google-Extended
150Disallow: /
151
152User-agent: Kimi-SearchBot
153Disallow: /
154
155User-agent: Kimi-User
156Disallow: /
157
158User-agent: KimiBot
159Disallow: /
160
161#Crawler
162
163User-agent: Alexibot
164Disallow: /
165
166User-agent: AlvinetSpider
167Disallow: /
168
169User-agent: AmiSoftware
170Disallow: /
171
172User-agent: Antenne Hatena
173Disallow: /
174
175User-agent: ApifyBot
176Disallow: /
177
178User-agent: ApifyWebsiteContentCrawler
179Disallow: /
180
181User-agent: ApocalXExplorerBot
182Disallow: /
183
184User-agent: Argus
185Disallow: /
186
187User-agent: Ask n read
188Disallow: /
189
190User-agent: asknread.com
191Disallow: /
192
193User-agent: asterias
194Disallow: /
195
196User-agent: BlowFish/1.0
197Disallow: /
198
199User-agent: BotALot
200Disallow: /
201
202User-agent: Brightbot
203Disallow: /
204
205User-agent: BuiltBotTough
206Disallow: /
207
208User-agent: Bullseye/1.0
209Disallow: /
210
211User-agent: BunnySlippers
212Disallow: /
213
214User-agent: Cegbfeieh
215Disallow: /
216
217User-agent: CheeseBot
218Disallow: /
219
220User-agent: ConveraCrawler
221Disallow: /
222
223User-agent: cosmos
224Disallow: /
225
226User-agent: CragCrawler
227Disallow: /
228
229User-agent: Crescent
230Disallow: /
231
232User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
233Disallow: /
234
235User-agent: cydralspider
236Disallow: /
237
238User-agent: Diffbot
239Disallow: /
240
241User-agent: Diffbot-User
242Disallow: /
243
244User-agent: DISCo Pump 3.1
245Disallow: /
246
247User-agent: DittoSpyder
248Disallow: /
249
250User-agent: EroCrawler
251Disallow: /
252
253User-agent: eureka
254Disallow: /
255
256User-agent: ExaBot
257Disallow: /
258
259User-agent: Explore
260Disallow: /
261
262User-agent: Fetch
263Disallow: /
264
265User-agent: FirecrawlAgent
266Disallow: /
267
268User-agent: Flamingo_SearchEngine
269Disallow: /
270
271User-agent: Foobot
272Disallow: /
273
274User-agent: gammaSpider
275Disallow: /
276
277User-agent: GeistHaus-PageFetcher
278Disallow: /
279
280User-agent: grub-client
281Disallow: /
282
283User-agent: hloader
284Disallow: /
285
286User-agent: httplib
287Disallow: /
288
289User-agent: humanlinks
290Disallow: /
291
292User-agent: ia_archiver
293Allow: /$
294Disallow: /
295
296User-agent: ia_archiver-web.archive.org
297Allow: /$
298Disallow: /
299
300User-agent: indexer
301Disallow: /
302
303User-agent: InfoNaviRobot
304Disallow: /
305
306User-agent: infoseek
307Disallow: /
308
309User-agent: JennyBot
310Disallow: /
311
312User-agent: Jetbot
313Disallow: /
314
315User-agent: JikeSpider
316Disallow: /
317
318User-agent: k2spider
319Disallow: /
320
321User-agent: Kenjin Spider
322Disallow: /
323
324User-agent: larbin
325Disallow: /
326
327User-agent: LexiBot
328Disallow: /
329
330User-agent: libWeb/clsHTTP
331Disallow: /
332
333User-agent: libwww
334Disallow: /
335
336User-agent: linko
337Disallow: /
338
339User-agent: LinkScan/8.1a Unix
340Disallow: /
341
342User-agent: LinkWalker
343Disallow: /
344
345User-agent: lwp-trivial
346Disallow: /
347
348User-agent: lwp-trivial/1.34
349Disallow: /
350
351User-agent: Mata Hari
352Disallow: /
353
354User-agent: Mister PiX
355Disallow: /
356
357User-agent: MLBot
358Disallow: /
359
360User-agent: moget
361Disallow: /
362
363User-agent: moget/2.1
364Disallow: /
365
366User-agent: Naverbot
367Disallow: /
368
369User-agent: NetAttache
370Disallow: /
371
372User-agent: Newzbin
373Disallow: /
374
375User-agent: NICErsPRO
376Disallow: /
377
378User-agent: NPBot
379Disallow: /
380
381User-agent: ObjectsSearch
382Disallow: /
383
384User-agent: omgili
385Disallow: /
386
387User-agent: omgilibot
388Disallow: /
389
390User-agent: Openfind
391Disallow: /
392
393User-agent: OpenindexSpider
394Disallow: /
395
396User-agent: Pimptrain
397Disallow: /
398
399User-agent: ProPowerBot/2.14
400Disallow: /
401
402User-agent: ProWebWalker
403Disallow: /
404
405User-agent: psbot
406Disallow: /
407
408User-agent: QuepasaCreep
409Disallow: /
410
411User-agent: Querit-SearchBot
412Disallow: /
413
414User-agent: QueritBot
415Disallow: /
416
417User-agent: QueryN Metasearch
418Disallow: /
419
420User-agent: Raven
421Disallow: /
422
423User-agent: RepoMonkey
424Disallow: /
425
426User-agent: RMA
427Disallow: /
428
429User-agent: Scrapy
430Disallow: /
431
432User-agent: ShapBot
433Disallow: /
434
435User-agent: SiteBot
436Disallow: /
437
438User-agent: Sogou web spider
439Disallow: /
440
441User-agent: sosospider
442Disallow: /
443
444User-agent: SpankBot
445Disallow: /
446
447User-agent: Speedy
448Disallow: /
449
450User-agent: spotter
451Disallow: /
452
453User-agent: suggybot
454Disallow: /
455
456User-agent: SuperBot
457Disallow: /
458
459User-agent: SuperBot/2.6
460Disallow: /
461
462User-agent: suzuran
463Disallow: /
464
465User-agent: Szukacz/1.4
466Disallow: /
467
468User-agent: Telesoft
469Disallow: /
470
471User-agent: The Intraformant
472Disallow: /
473
474User-agent: TheNomad
475Disallow: /
476
477User-agent: TightTwatBot
478Disallow: /
479
480User-agent: Titan
481Disallow: /
482
483User-agent: toCrawl/UrlDispatcher
484Disallow: /
485
486User-agent: TosCrawler
487Disallow: /
488
489User-agent: True_Robot
490Disallow: /
491
492User-agent: True_Robot/1.0
493Disallow: /
494
495User-agent: turingos
496Disallow: /
497
498User-agent: URLy Warning
499Disallow: /
500
501User-agent: VCI
502Disallow: /
503
504User-agent: vsw
505Disallow: /
506
507User-agent: wapspider
508Disallow: /
509
510User-agent: WebBandit
511Disallow: /
512
513User-agent: WebBandit/3.50
514Disallow: /
515
516User-agent: WebEnhancer
517Disallow: /
518
519User-agent: WebmasterWorldForumBot
520Disallow: /
521
522User-agent: Webster Pro
523Disallow: /
524
525User-agent: WebZinger
526Disallow: /
527
528User-agent: Webzio-Extended
529Disallow: /
530
531User-agent: WWW-Collector-E
532Disallow: /
533
534User-agent: yacy
535Disallow: /
536
537User-agent: yandex
538Disallow: /
539
540User-agent: YRSPider
541Disallow: /
542
543User-agent: Zealbot
544Disallow: /
545
546User-agent: Zeus
547Disallow: /
548
549User-agent: Zite
550Disallow: /
551
552User-agent: Zookabot
553Disallow: /
554
555User-agent: ZyBORG
556Disallow: /
557
558#Outil
559
560User-agent: 5emeRue
561Disallow: /
562
563User-agent: 5erue
564Disallow: /
565
566User-agent: adequat
567Disallow: /
568
569User-agent: adequat-systems
570Disallow: /
571
572User-agent: Amazonbot
573Disallow: /
574
575User-agent: Augure
576Disallow: /
577
578User-agent: auramundi
579Disallow: /
580
581User-agent: BizInformation
582Disallow: /
583
584User-agent: Bloomberg
585Disallow: /
586
587User-agent: CherryPicker
588Disallow: /
589
590User-agent: CherryPickerElite/1.0
591Disallow: /
592
593User-agent: CherryPickerSE/1.0
594Disallow: /
595
596User-agent: Cision
597Disallow: /
598
599User-agent: coexel
600Disallow: /
601
602User-agent: CopyRightCheck
603Disallow: /
604
605User-agent: Corporama
606Disallow: /
607
608User-agent: Digimind
609Disallow: /
610
611User-agent: dotbot
612Disallow: /
613
614User-agent: Download Ninja
615Disallow: /
616
617User-agent: downloadexpress
618Disallow: /
619
620User-agent: ellisphere
621Disallow: /
622
623User-agent: EmailCollector
624Disallow: /
625
626User-agent: EmailSiphon
627Disallow: /
628
629User-agent: EmailWolf
630Disallow: /
631
632User-agent: Europresse
633Disallow: /
634
635User-agent: ExtractorPro
636Disallow: /
637
638User-agent: Fasterfox
639Disallow: /
640
641User-agent: HTTrack
642Disallow: /
643
644User-agent: HTTrack 3.0
645Disallow: /
646
647User-agent: Igentia
648Disallow: /
649
650User-agent: Kantar
651Disallow: /
652
653User-agent: kbcrawl
654Disallow: /
655
656User-agent: Knowings
657Disallow: /
658
659User-agent: leadbox
660Disallow: /
661
662User-agent: LinkextractorPro
663Disallow: /
664
665User-agent: linkfluence
666Disallow: /
667
668User-agent: manageo
669Disallow: /
670
671User-agent: mediacompil
672Disallow: /
673
674User-agent: mention
675Disallow: /
676
677User-agent: Moreover
678Disallow: /
679
680User-agent: mytwip
681Disallow: /
682
683User-agent: NetAnts
684Disallow: /
685
686User-agent: NetMechanic
687Disallow: /
688
689User-agent: newscan-online
690Disallow: /
691
692User-agent: NewsNow
693Disallow: /
694
695User-agent: Offline Explorer
696Disallow: /
697
698User-agent: opinion-tracker
699Disallow: /
700
701User-agent: Qwam content intelligence
702Disallow: /
703
704User-agent: readability.com
705Disallow: /
706
707User-agent: scoop.it
708Disallow: /
709
710User-agent: score3
711Disallow: /
712
713User-agent: SemrushBot
714Disallow: /
715
716User-agent: SightupBot
717Disallow: /
718
719User-agent: Sindup
720Disallow: /
721
722User-agent: SirdataBot
723Disallow: /
724
725User-agent: SiteSnagger
726Disallow: /
727
728User-agent: SiteSucker
729Disallow: /
730
731User-agent: Synthesio
732Disallow: /
733
734User-agent: Talkwalker
735Disallow: /
736
737User-agent: TavilyBot
738Disallow: /
739
740User-agent: Teleport
741Disallow: /
742
743User-agent: TeleportPro
744Disallow: /
745
746User-agent: trendeo
747Disallow: /
748
749User-agent: trendybuzz
750Disallow: /
751
752User-agent: TurnitinBot
753Disallow: /
754
755User-agent: up2news
756Disallow: /
757
758User-agent: UrlPouls
759Disallow: /
760
761User-agent: UseAI
762Disallow: /
763
764User-agent: vecteurplus
765Disallow: /
766
767User-agent: Verif
768Disallow: /
769
770User-agent: verticalsearch
771Disallow: /
772
773User-agent: Web Image Collector
774Disallow: /
775
776User-agent: WebAuto
777Disallow: /
778
779User-agent: WebCopier
780Disallow: /
781
782User-agent: webcopy
783Disallow: /
784
785User-agent: Webedia
786Disallow: /
787
788User-agent: webmirror
789Disallow: /
790
791User-agent: WebReaper
792Disallow: /
793
794User-agent: WebSauger
795Disallow: /
796
797User-agent: website extractor
798Disallow: /
799
800User-agent: Website Quester
801Disallow: /
802
803User-agent: WebStripper
804Disallow: /
805
806User-agent: WebStripper/2.02
807Disallow: /
808
809User-agent: WebZIP
810Disallow: /
811
812User-agent: Wget
813Disallow: /
814
815User-agent: WikioFeedBot
816Disallow: /
817
818User-agent: winello
819Disallow: /
820
821User-agent: WinHTTrack
822Disallow: /
823
824User-agent: Xenu Link Sleuth/1.3.8
825Disallow: /
826
827User-agent: YouBot
828Disallow: /
829
830User-agent: Youmag
831Disallow: /
832
833Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/articles.xml
834Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/sections.xml
835Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/topics.xml
836Sitemap: https://www.lefigaro.fr/sitemap_news.xml
837Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/elections/resultats/sitemap.xml
838Sitemap: https://static.lefigaro.fr/f1/lefigaro/sitemaps/robots-service.txt