NeuralCrawl

EDF / robots.txt

← back to edf.fr · change detected 1mo ago (2026-09-30T19:18:16Z) · modified +86 −40

What changed

Diff old (2026-06-20T01:10:30Z) → new (2026-09-30T19:18:16Z)

@@ -1,18 +1,3 @@
1- #
2- # robots.txt
3- #
4- # This file is to prevent the crawling and indexing of certain parts
5- # of your site by web crawlers and spiders run by sites like Yahoo!
6- # and Google. By telling these "robots" where not to go on your site,
7- # you save bandwidth and server resources.
8- #
9- # This file will be ignored unless it is at the root of your host:
10- # Used: http://example.com/robots.txt
11- # Ignored: http://example.com/site/robots.txt
12- #
13- # For more information about the robots.txt standard, see:
14- # http://www.robotstxt.org/robotstxt.html
15-
161  User-agent: *
172  # CSS, JS, Images
183  Allow: /core/*.css$
@@ -66,34 +51,95 @@
6651  Disallow: /resultats?keywords=*
6752  Disallow: /en/search-results?keywords=*
6853  Disallow: /edf-recrute/offre/detail/*/postuler
69- Disallow: /en/edf-join-us/offer/detail/*/apply
70-
71- # Ajout 11/09/24 suite à Audit SEO
54+ Disallow: /en/edf-join-us/offer/detail/*/apply
55+ # Budget Crawl
7256  Disallow: /de/tous-les-dossiers-de-presse/feed?mots_cles=*
7357  Disallow: /tous-les-dossiers-de-presse/feed?mots_cles=*
74- Disallow: /en/tous-les-dossiers-de-presse/feed?mots_cles=*"
75-
76- Sitemap: https://www.edf.fr/sitemap.xml
77-
78- # Mr Roger Bot
79- User-agent: rogerbot
80- Disallow: /
81-
82- # BLEXBot
83- User-agent: BLEXBot
84- Disallow: /faq/add/feedBack/
85-
86- Crawl-delay: 10
87-
88- Disallow: /includes/
89- Disallow: /misc/
90- Disallow: /modules/
91-
92- Disallow: /LICENSE.txt
93- Disallow: /update.php
94- Disallow: /xmlrpc.php
95-
58+ Disallow: /en/tous-les-dossiers-de-presse/feed?mots_cles=*
9659  Disallow: /groupe-edf/qui-sommes-nous/activites/optimisation-et-trading/liste-des-indisponibilites-et-des-messages/liste-des-indisponibilites/05470-*
9760  Disallow: /groupe-edf/qui-sommes-nous/activites/optimisation-et-trading/liste-des-indisponibilites-et-des-messages/liste-des-messages/EDF-*
9861  Disallow: /groupe-edf/ambition-neutralite-co2-pour-edf-a-l-horizon-2050/optimisation-et-trading/listes-des-indisponibilites-et-des-messages/liste-des-indisponibilites?*
9962  Disallow: /groupe-edf/ambition-neutralite-co2-pour-edf-a-l-horizon-2050/optimisation-et-trading/listes-des-indisponibilites-et-des-messages/liste-des-messages?*
63+ # Robots IA autorisés explicitement, mêmes règles que le bloc principal
64+ User-agent: GPTBot
65+ User-agent: OAI-SearchBot
66+ User-agent: ChatGPT-User
67+ User-agent: ClaudeBot
68+ User-agent: Claude-SearchBot
69+ User-agent: Claude-User
70+ User-agent: anthropic-ai
71+ User-agent: claude-web
72+ User-agent: PerplexityBot
73+ User-agent: Perplexity-User
74+ User-agent: Google-Extended
75+ User-agent: Applebot
76+ User-agent: Applebot-Extended
77+ User-agent: Amazonbot
78+ User-agent: Meta-ExternalAgent
79+ User-agent: CCBot
80+ User-agent: MistralAI-User
81+ # CSS, JS, Images
82+ Allow: /core/*.css$
83+ Allow: /core/*.css?
84+ Allow: /core/*.js$
85+ Allow: /core/*.js?
86+ Allow: /core/*.gif
87+ Allow: /core/*.jpg
88+ Allow: /core/*.jpeg
89+ Allow: /core/*.png
90+ Allow: /core/*.svg
91+ Allow: /profiles/*.css$
92+ Allow: /profiles/*.css?
93+ Allow: /profiles/*.js$
94+ Allow: /profiles/*.js?
95+ Allow: /profiles/*.gif
96+ Allow: /profiles/*.jpg
97+ Allow: /profiles/*.jpeg
98+ Allow: /profiles/*.png
99+ Allow: /profiles/*.svg
100+ # Directories
101+ Disallow: /core/
102+ Disallow: /profiles/
103+ # Files
104+ Disallow: /README.txt
105+ Disallow: /web.config
106+ # Paths (clean URLs)
107+ Disallow: /admin/
108+ Disallow: /comment/reply/
109+ Disallow: /filter/tips
110+ Disallow: /node/add/
111+ Disallow: /search/
112+ Disallow: /user/register
113+ Disallow: /user/password
114+ Disallow: /user/login
115+ Disallow: /user/logout
116+ Disallow: /media/oembed
117+ Disallow: /*/media/oembed
118+ # Paths (no clean URLs)
119+ Disallow: /index.php/admin/
120+ Disallow: /index.php/comment/reply/
121+ Disallow: /index.php/filter/tips
122+ Disallow: /index.php/node/add/
123+ Disallow: /index.php/search/
124+ Disallow: /index.php/user/password
125+ Disallow: /index.php/user/register
126+ Disallow: /index.php/user/login
127+ Disallow: /index.php/user/logout
128+ Disallow: /index.php/media/oembed
129+ Disallow: /index.php/*/media/oembed
130+ Disallow: /resultats?keywords=*
131+ Disallow: /en/search-results?keywords=*
132+ Disallow: /edf-recrute/offre/detail/*/postuler
133+ Disallow: /en/edf-join-us/offer/detail/*/apply
134+ # Budget Crawl
135+ Disallow: /de/tous-les-dossiers-de-presse/feed?mots_cles=*
136+ Disallow: /tous-les-dossiers-de-presse/feed?mots_cles=*
137+ Disallow: /en/tous-les-dossiers-de-presse/feed?mots_cles=*
138+ Disallow: /groupe-edf/qui-sommes-nous/activites/optimisation-et-trading/liste-des-indisponibilites-et-des-messages/liste-des-indisponibilites/05470-*
139+ Disallow: /groupe-edf/qui-sommes-nous/activites/optimisation-et-trading/liste-des-indisponibilites-et-des-messages/liste-des-messages/EDF-*
140+ Disallow: /groupe-edf/ambition-neutralite-co2-pour-edf-a-l-horizon-2050/optimisation-et-trading/listes-des-indisponibilites-et-des-messages/liste-des-indisponibilites?*
141+ Disallow: /groupe-edf/ambition-neutralite-co2-pour-edf-a-l-horizon-2050/optimisation-et-trading/listes-des-indisponibilites-et-des-messages/liste-des-messages?*
142+ # Mr Roger Bot
143+ User-agent: rogerbot
144+ Disallow: /
145+ Sitemap: https://www.edf.fr/sitemap.xml