NeuralCrawl

Le Figaro / robots.txt snapshot

← back to lefigaro.fr · fetched 2026-07-20T19:11:56Z (16d ago) · HTTP 200 · 12528 bytes · sha256 90c39482ca2e159f · raw

final URL: https://www.lefigaro.fr/robots.txt

1# :.
2# ++.
3# .+**:
4# .+****.
5# =******..
6# .********:.
7# -*********-.
8# =***********#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
9# .-************#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
10# .++=*************#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
11# .+****************FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF#.
12# -****************#FFFFFFFF# .#FFFFFFFF#.
13# .+***************#FFFFFFF# .=+++++++++.
14# .+**************#FFFFFF#
15# .-+************FFFFFF#
16# .-**************FFFFFFFFFFFFFFFFF:
17# .-************FFFFFFFFFFFFFFFFF:
18# ..=*********FFFFFFFFFFFFFFFFF:
19# .##******FFFFFFFFFFFFFFFFF:
20# #FFFF##*FFFFFF#*+.
21# #FFFFFFFFFFFFF#***=.
22# #FFFFFFFFFFFFF# .:+*-.
23# .#FFFFFFFFFFFFF#. -*-
24# +FFFFFFFFFFFFFFFFFFFFFFF- :+.
25# +FFFFFFFFFFFFFFFFFFFFFFF- .-=.
26# +FFFFFFFFFFFFFFFFFFFFFFF- .=-.
27# +FFFFFFFFFFFFFFFFFFFFFFF- .+.
28
29# Toute utilisation de nos contenus protégés autre qu'un usage strictement individuel (incluant notamment l'entraînement des grands modèles de langage (LLM),
30# l'entraînement des outils d'intelligence artificielle, la veille web ou le media monitoring) est conditionnée à la conclusion d'une licence d'utilisation
31# avec la SOCIETE DU FIGARO. Nous vous invitons à cette fin à contacter [email protected].
32#
33# Toute utilisation non-autorisée de nos contenus protégés est constitutive d'une contrefaçon de droit d'auteur et/ou du droit de producteur
34# de base de données et susceptible d'être poursuivie.
35#
36# Il est interdit de crawler notre site web en utilisant un agent d'utilisateur (user agent) volé qui ne correspond pas à votre identité.
37# L'utilisation des robots d'indexation web ou d'autres méthodes automatiques de feuilletage ou de navigation sur ce site Web n'est pas autorisée.
38
39#Généralités
40
41User-agent: *
42Disallow: /async/
43Disallow: /*?noheader*
44Disallow: /*?*&noheader*
45Disallow: /*?sfdebug*
46Disallow: /*?*&sfdebug*
47Disallow: /brouillon/
48Disallow: /proxy/
49Disallow: /synsearch/
50Disallow: /a-savoir-en-france/
51Disallow: /vos-questions
52
53#LLM
54
55User-agent: ChatGPT-User
56Allow: /voyages
57Allow: /culture
58Allow: /style
59Allow: /beaute
60Allow: /bons-plans
61Allow: /elections/resultats
62Disallow: /
63
64User-agent: Claude-SearchBot
65Allow: /voyages
66Allow: /bons-plans
67Allow: /elections/resultats
68Disallow: /
69
70User-agent: Claude-Web
71Allow: /voyages
72Allow: /bons-plans
73Allow: /elections/resultats
74Disallow: /
75
76User-agent: OAI-SearchBot
77Allow: /voyages
78Allow: /culture
79Allow: /style
80Allow: /beaute
81Allow: /bons-plans
82Allow: /elections/resultats
83Disallow: /
84
85User-agent: Claude-User
86Allow: /voyages
87Allow: /bons-plans
88Allow: /elections/resultats
89Disallow: /
90
91User-agent: ChatGPT Agent
92Allow: /voyages
93Allow: /bons-plans
94Allow: /elections/resultats
95Disallow: /
96
97User-agent: GoogleAgent-Mariner
98Allow: /voyages
99Allow: /bons-plans
100Allow: /elections/resultats
101Disallow: /
102
103User-agent: Google-NotebookLM
104Allow: /voyages
105Allow: /bons-plans
106Allow: /elections/resultats
107Disallow: /
108
109User-agent: Google-CloudVertexBot
110Allow: /voyages
111Allow: /bons-plans
112Allow: /elections/resultats
113Disallow: /
114
115User-agent: MistralAI-User
116Allow: /voyages
117Allow: /bons-plans
118Allow: /elections/resultats
119Disallow: /
120
121User-agent: Gemini-Deep-Research
122Allow: /voyages
123Allow: /bons-plans
124Allow: /elections/resultats
125Disallow: /
126
127User-agent: CloudVertexBot
128Disallow: /
129
130User-agent: anthropic-ai
131Disallow: /
132
133User-agent: Bytespider
134Disallow: /
135
136User-agent: CCBot
137Disallow: /
138
139User-agent: ClaudeBot
140Disallow: /
141
142User-agent: cohere-ai
143Disallow: /
144
145User-agent: CohereBot
146Disallow: /
147
148User-agent: GPTBot
149Disallow: /
150
151User-agent: Google-Extended
152Disallow: /
153
154User-agent: Kimi-SearchBot
155Disallow: /
156
157User-agent: Kimi-User
158Disallow: /
159
160User-agent: KimiBot
161Disallow: /
162
163#Crawler
164
165User-agent: Alexibot
166Disallow: /
167
168User-agent: AlvinetSpider
169Disallow: /
170
171User-agent: AmiSoftware
172Disallow: /
173
174User-agent: Antenne Hatena
175Disallow: /
176
177User-agent: ApifyBot
178Disallow: /
179
180User-agent: ApifyWebsiteContentCrawler
181Disallow: /
182
183User-agent: ApocalXExplorerBot
184Disallow: /
185
186User-agent: Argus
187Disallow: /
188
189User-agent: Ask n read
190Disallow: /
191
192User-agent: asknread.com
193Disallow: /
194
195User-agent: asterias
196Disallow: /
197
198User-agent: BlowFish/1.0
199Disallow: /
200
201User-agent: BotALot
202Disallow: /
203
204User-agent: Brightbot
205Disallow: /
206
207User-agent: BuiltBotTough
208Disallow: /
209
210User-agent: Bullseye/1.0
211Disallow: /
212
213User-agent: BunnySlippers
214Disallow: /
215
216User-agent: Cegbfeieh
217Disallow: /
218
219User-agent: CheeseBot
220Disallow: /
221
222User-agent: ConveraCrawler
223Disallow: /
224
225User-agent: cosmos
226Disallow: /
227
228User-agent: CragCrawler
229Disallow: /
230
231User-agent: Crescent
232Disallow: /
233
234User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
235Disallow: /
236
237User-agent: cydralspider
238Disallow: /
239
240User-agent: Diffbot
241Disallow: /
242
243User-agent: Diffbot-User
244Disallow: /
245
246User-agent: DISCo Pump 3.1
247Disallow: /
248
249User-agent: DittoSpyder
250Disallow: /
251
252User-agent: EroCrawler
253Disallow: /
254
255User-agent: eureka
256Disallow: /
257
258User-agent: ExaBot
259Disallow: /
260
261User-agent: Explore
262Disallow: /
263
264User-agent: Fetch
265Disallow: /
266
267User-agent: FirecrawlAgent
268Disallow: /
269
270User-agent: Flamingo_SearchEngine
271Disallow: /
272
273User-agent: Foobot
274Disallow: /
275
276User-agent: gammaSpider
277Disallow: /
278
279User-agent: GeistHaus-PageFetcher
280Disallow: /
281
282User-agent: grub-client
283Disallow: /
284
285User-agent: hloader
286Disallow: /
287
288User-agent: httplib
289Disallow: /
290
291User-agent: humanlinks
292Disallow: /
293
294User-agent: ia_archiver
295Allow: /$
296Disallow: /
297
298User-agent: ia_archiver-web.archive.org
299Allow: /$
300Disallow: /
301
302User-agent: indexer
303Disallow: /
304
305User-agent: InfoNaviRobot
306Disallow: /
307
308User-agent: infoseek
309Disallow: /
310
311User-agent: JennyBot
312Disallow: /
313
314User-agent: Jetbot
315Disallow: /
316
317User-agent: JikeSpider
318Disallow: /
319
320User-agent: k2spider
321Disallow: /
322
323User-agent: Kenjin Spider
324Disallow: /
325
326User-agent: larbin
327Disallow: /
328
329User-agent: LexiBot
330Disallow: /
331
332User-agent: libWeb/clsHTTP
333Disallow: /
334
335User-agent: libwww
336Disallow: /
337
338User-agent: linko
339Disallow: /
340
341User-agent: LinkScan/8.1a Unix
342Disallow: /
343
344User-agent: LinkWalker
345Disallow: /
346
347User-agent: lwp-trivial
348Disallow: /
349
350User-agent: lwp-trivial/1.34
351Disallow: /
352
353User-agent: Mata Hari
354Disallow: /
355
356User-agent: Mister PiX
357Disallow: /
358
359User-agent: MLBot
360Disallow: /
361
362User-agent: moget
363Disallow: /
364
365User-agent: moget/2.1
366Disallow: /
367
368User-agent: Naverbot
369Disallow: /
370
371User-agent: NetAttache
372Disallow: /
373
374User-agent: Newzbin
375Disallow: /
376
377User-agent: NICErsPRO
378Disallow: /
379
380User-agent: NPBot
381Disallow: /
382
383User-agent: ObjectsSearch
384Disallow: /
385
386User-agent: omgili
387Disallow: /
388
389User-agent: omgilibot
390Disallow: /
391
392User-agent: Openfind
393Disallow: /
394
395User-agent: OpenindexSpider
396Disallow: /
397
398User-agent: Pimptrain
399Disallow: /
400
401User-agent: ProPowerBot/2.14
402Disallow: /
403
404User-agent: ProWebWalker
405Disallow: /
406
407User-agent: psbot
408Disallow: /
409
410User-agent: QuepasaCreep
411Disallow: /
412
413User-agent: Querit-SearchBot
414Disallow: /
415
416User-agent: QueritBot
417Disallow: /
418
419User-agent: QueryN Metasearch
420Disallow: /
421
422User-agent: Raven
423Disallow: /
424
425User-agent: RepoMonkey
426Disallow: /
427
428User-agent: RMA
429Disallow: /
430
431User-agent: Scrapy
432Disallow: /
433
434User-agent: ShapBot
435Disallow: /
436
437User-agent: SiteBot
438Disallow: /
439
440User-agent: Sogou web spider
441Disallow: /
442
443User-agent: sosospider
444Disallow: /
445
446User-agent: SpankBot
447Disallow: /
448
449User-agent: Speedy
450Disallow: /
451
452User-agent: spotter
453Disallow: /
454
455User-agent: suggybot
456Disallow: /
457
458User-agent: SuperBot
459Disallow: /
460
461User-agent: SuperBot/2.6
462Disallow: /
463
464User-agent: suzuran
465Disallow: /
466
467User-agent: Szukacz/1.4
468Disallow: /
469
470User-agent: Telesoft
471Disallow: /
472
473User-agent: The Intraformant
474Disallow: /
475
476User-agent: TheNomad
477Disallow: /
478
479User-agent: TightTwatBot
480Disallow: /
481
482User-agent: Titan
483Disallow: /
484
485User-agent: toCrawl/UrlDispatcher
486Disallow: /
487
488User-agent: TosCrawler
489Disallow: /
490
491User-agent: True_Robot
492Disallow: /
493
494User-agent: True_Robot/1.0
495Disallow: /
496
497User-agent: turingos
498Disallow: /
499
500User-agent: URLy Warning
501Disallow: /
502
503User-agent: VCI
504Disallow: /
505
506User-agent: vsw
507Disallow: /
508
509User-agent: wapspider
510Disallow: /
511
512User-agent: WebBandit
513Disallow: /
514
515User-agent: WebBandit/3.50
516Disallow: /
517
518User-agent: WebEnhancer
519Disallow: /
520
521User-agent: WebmasterWorldForumBot
522Disallow: /
523
524User-agent: Webster Pro
525Disallow: /
526
527User-agent: WebZinger
528Disallow: /
529
530User-agent: Webzio-Extended
531Disallow: /
532
533User-agent: WWW-Collector-E
534Disallow: /
535
536User-agent: yacy
537Disallow: /
538
539User-agent: yandex
540Disallow: /
541
542User-agent: YRSPider
543Disallow: /
544
545User-agent: Zealbot
546Disallow: /
547
548User-agent: Zeus
549Disallow: /
550
551User-agent: Zite
552Disallow: /
553
554User-agent: Zookabot
555Disallow: /
556
557User-agent: ZyBORG
558Disallow: /
559
560#Outil
561
562User-agent: 5emeRue
563Disallow: /
564
565User-agent: 5erue
566Disallow: /
567
568User-agent: adequat
569Disallow: /
570
571User-agent: adequat-systems
572Disallow: /
573
574User-agent: Amazonbot
575Disallow: /
576
577User-agent: Augure
578Disallow: /
579
580User-agent: auramundi
581Disallow: /
582
583User-agent: BizInformation
584Disallow: /
585
586User-agent: Bloomberg
587Disallow: /
588
589User-agent: CherryPicker
590Disallow: /
591
592User-agent: CherryPickerElite/1.0
593Disallow: /
594
595User-agent: CherryPickerSE/1.0
596Disallow: /
597
598User-agent: Cision
599Disallow: /
600
601User-agent: coexel
602Disallow: /
603
604User-agent: CopyRightCheck
605Disallow: /
606
607User-agent: Corporama
608Disallow: /
609
610User-agent: Digimind
611Disallow: /
612
613User-agent: dotbot
614Disallow: /
615
616User-agent: Download Ninja
617Disallow: /
618
619User-agent: downloadexpress
620Disallow: /
621
622User-agent: ellisphere
623Disallow: /
624
625User-agent: EmailCollector
626Disallow: /
627
628User-agent: EmailSiphon
629Disallow: /
630
631User-agent: EmailWolf
632Disallow: /
633
634User-agent: Europresse
635Disallow: /
636
637User-agent: ExtractorPro
638Disallow: /
639
640User-agent: Fasterfox
641Disallow: /
642
643User-agent: HTTrack
644Disallow: /
645
646User-agent: HTTrack 3.0
647Disallow: /
648
649User-agent: Igentia
650Disallow: /
651
652User-agent: Kantar
653Disallow: /
654
655User-agent: kbcrawl
656Disallow: /
657
658User-agent: Knowings
659Disallow: /
660
661User-agent: leadbox
662Disallow: /
663
664User-agent: LinkextractorPro
665Disallow: /
666
667User-agent: linkfluence
668Disallow: /
669
670User-agent: manageo
671Disallow: /
672
673User-agent: mediacompil
674Disallow: /
675
676User-agent: mention
677Disallow: /
678
679User-agent: Moreover
680Disallow: /
681
682User-agent: mytwip
683Disallow: /
684
685User-agent: NetAnts
686Disallow: /
687
688User-agent: NetMechanic
689Disallow: /
690
691User-agent: newscan-online
692Disallow: /
693
694User-agent: NewsNow
695Disallow: /
696
697User-agent: Offline Explorer
698Disallow: /
699
700User-agent: opinion-tracker
701Disallow: /
702
703User-agent: Qwam content intelligence
704Disallow: /
705
706User-agent: readability.com
707Disallow: /
708
709User-agent: scoop.it
710Disallow: /
711
712User-agent: score3
713Disallow: /
714
715User-agent: SemrushBot
716Disallow: /
717
718User-agent: SightupBot
719Disallow: /
720
721User-agent: Sindup
722Disallow: /
723
724User-agent: SirdataBot
725Disallow: /
726
727User-agent: SiteSnagger
728Disallow: /
729
730User-agent: SiteSucker
731Disallow: /
732
733User-agent: Synthesio
734Disallow: /
735
736User-agent: Talkwalker
737Disallow: /
738
739User-agent: TavilyBot
740Disallow: /
741
742User-agent: Teleport
743Disallow: /
744
745User-agent: TeleportPro
746Disallow: /
747
748User-agent: trendeo
749Disallow: /
750
751User-agent: trendybuzz
752Disallow: /
753
754User-agent: TurnitinBot
755Disallow: /
756
757User-agent: up2news
758Disallow: /
759
760User-agent: UrlPouls
761Disallow: /
762
763User-agent: UseAI
764Disallow: /
765
766User-agent: vecteurplus
767Disallow: /
768
769User-agent: Verif
770Disallow: /
771
772User-agent: verticalsearch
773Disallow: /
774
775User-agent: Web Image Collector
776Disallow: /
777
778User-agent: WebAuto
779Disallow: /
780
781User-agent: WebCopier
782Disallow: /
783
784User-agent: webcopy
785Disallow: /
786
787User-agent: Webedia
788Disallow: /
789
790User-agent: webmirror
791Disallow: /
792
793User-agent: WebReaper
794Disallow: /
795
796User-agent: WebSauger
797Disallow: /
798
799User-agent: website extractor
800Disallow: /
801
802User-agent: Website Quester
803Disallow: /
804
805User-agent: WebStripper
806Disallow: /
807
808User-agent: WebStripper/2.02
809Disallow: /
810
811User-agent: WebZIP
812Disallow: /
813
814User-agent: Wget
815Disallow: /
816
817User-agent: WikioFeedBot
818Disallow: /
819
820User-agent: winello
821Disallow: /
822
823User-agent: WinHTTrack
824Disallow: /
825
826User-agent: Xenu Link Sleuth/1.3.8
827Disallow: /
828
829User-agent: YouBot
830Disallow: /
831
832User-agent: Youmag
833Disallow: /
834
835Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/articles.xml
836Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/sections.xml
837Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/topics.xml
838Sitemap: https://www.lefigaro.fr/sitemap_news.xml
839Sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/elections/resultats/sitemap.xml
840Sitemap: https://static.lefigaro.fr/f1/lefigaro/sitemaps/robots-service.txt