NeuralCrawl

Corriere della Sera / robots.txt snapshot

← back to corriere.it · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 9377 bytes · sha256 8ea78819373f92a7 · raw

final URL: https://www.corriere.it/robots.txt

1# Inizio Bot Exclusion Protocol
2User-agent: GPTBot
3Disallow: /
4User-agent: CCBot
5Disallow: /
6User-agent: anthropic-ai
7Disallow: /
8User-agent: AmazonAdBot
9Allow: /
10User-agent: Google-CloudVertexBot
11Disallow: /
12User-agent: AppleBot
13Disallow: /
14User-agent: Google-Extended
15Disallow: /
16User-agent: Applebot-Extended
17Disallow: /
18User-agent: ClaudeBot
19Disallow: /
20User-agent: Amazonbot
21Disallow: /
22User-agent: Meta-ExternalAgent
23Disallow: /
24User-agent: PetalBot
25Disallow: /
26User-agent: Yandex
27Disallow: / # prohibits crawling for the entire site
28# Fine Bot Exclusion Protocol
29
30User-agent: *
31Disallow: *{{data.c.link_offerId}}
32
33
34Disallow: *(none)
35Disallow: *?RedirectUrl
36Disallow: /privacy.shtml
37Disallow: /abbonati.Suspended/
38Disallow: /corcommunitynew/accesso/LogOut.do
39Disallow: /indexMethodeProve.shtml
40Disallow: /apw.js
41Disallow: /salute/cardiologia/sintomi-infartosintomi-infarto
42Disallow: /salute/cardiologia/sintomi-infartoheart-attack-symptoms
43Disallow: /salute/cardiologia/sintomi-infartosíntomas-ataque-cardíaco
44Disallow: /salute/cardiologia/sintomi-infartosymptomes-infarctus
45Disallow: /salute/cardiologia/sintomi-infartoсимптомы-инфаркта
46Disallow: /salute/cardiologia/sintomi-infarto心肌梗塞症状
47Disallow: /salute/cardiologia/sintomi-infartoطب القلب
48Disallow: /salute/cardiologia/sintomi-infartosimptome-de-atac-de-cord
49Disallow: /salute/sportello_cancro/jsp/*
50Disallow: /corrierecalcio/app/formazione/
51Disallow: /preview/
52Disallow: /dwc-xalok/
53Disallow: /ssi/
54Disallow: /tools/
55Disallow: /linked_webroots/
56Disallow: /english/includes_methode/
57Disallow: /includes_methode/
58Disallow: /community-corriere/
59Disallow: /widget/
60Disallow: /dwc/
61Disallow: /digital-edition/
62Disallow: /correlati/
63Disallow: /cinema/icortissimi-ended/
64Disallow: /ZCestino/
65Disallow: /prova/
66Disallow: /test/
67Disallow: /landing/wrapped-2023/
68Disallow: /test_mf/
69Disallow: /piano-511/
70Disallow: /b2b/
71Disallow: /_template/
72Disallow: /_docs/
73Disallow: /_temp/
74Disallow: /cf/
75Disallow: /servizi/
76Disallow: /salute/dizionario/dizionarioSearch.jsp
77Disallow: /sportello-cancro/dizionario/
78Disallow: /dilatua/proRecensione.do
79Disallow: /newsletter/nl_economia.html
80# vecchia ultima ora
81Disallow: /ultima_ora/
82#Dizionario della Salute
83Disallow: /salute/dizionario/dizionarioSearch.jsp
84Disallow: /salute/dizionario/pages/results.jsp
85Disallow: /cronache/speciali/2010/maturita/test-universita.shtml
86Disallow: /smallApp/tellafriend/
87Disallow: /*_print.html$
88Disallow: /*.xml
89Disallow: /account/
90Allow: /account/login$
91Disallow: /account/login?
92Disallow: /gallery/*/vuoto.shtml
93Disallow: *pwcmp=
94Disallow: *reader_id=
95Disallow: /cruciverba_old/
96Allow: /salute/il-medico-risponde/sitemap.xml
97Allow: /salute/il-medico-risponde/news/sitemap.xml
98Allow: /salute/sitemap-dizionario-corriere-salute.xml
99Allow: /dynamic-sitemap/sitemap-last-100/*.xml
100Allow: /dynamic-feed/*.xml
101Allow: /cook*sitemap*.xml
102Allow: /guida-autovelox-e-codice-della-strada/sitemap.xml
103Allow: /radio/$
104Allow: /radio/index.shtml
105Allow: /servizi/$
106Allow: /ultima_ora/agrnews.jsp
107Allow: /showcase/*.xml
108Allow: /economia/chiedi-esperto/sitemap.xml
109Allow: /economia/chiedi-esperto/news/sitemap.xml
110Allow: /sette/speciale-presidenziali-usa-2024-harris-contro-trump/rss/*.xml
111Allow: /dynamic-sitemap/corriere/video/lezioni*sitemap*.xml
112
113
114# item
115Disallow: /corrierepedia
116Disallow: /firme/pierluigi-battista_80
117Disallow: /cook/ricette/ricerca
118Disallow: /CorriereAppNews/
119Disallow: /corrierepoint/
120Disallow: /index_mobile_tst_hp_mob_b.shtml
121Disallow: /*app_v2
122Disallow: /*app_v1
123Allow: /sitemap*.xml
124# Allow: /notizie-ultima-ora/rss_col.xml
125Allow: /rss/*.xml
126Allow: /salute/sitemap-dizionario.xml
127Allow: /rss/ultimora.xml
128# Allow: /rss/homepage.xml
129Allow: /feed-hp/homepage-restyle-2025.xml
130# Allow: /notizie-ultima-ora/sitemap-news.xml
131#richiesta da Ruggiero BG27112011
132Allow: /editorspicks/
133Allow: /oroscopo/sitemap.xml
134Allow: /sport/risultati-live/sitemap.xml
135Allow: /elezioni/*.xml
136Disallow: /*?q=
137Disallow: /*?f=
138Disallow: /*?s=
139Disallow: /*?searchType=
140Disallow: /*articleurl=
141Disallow: /economia/quiz/
142Disallow: /offerta-speciale-limitata/
143Disallow: /*?*rid=*&*destination=*
144Disallow: /*?*destination=*&*rid=*
145Disallow: /nazionale-2025/
146
147Sitemap: https://www.corriere.it/feed-hp/homepage-restyle-2025.xml
148Sitemap: https://www.corriere.it/salute/sitemap-dizionario-corriere-salute.xml
149Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/video/Corriere.xml
150Sitemap: https://www.corriere.it/rss/sitemap_v2.xml
151Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Economia.xml
152Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Salute.xml
153Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Scienze.xml
154Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Interni.xml
155Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Esteri.xml
156Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Sport.xml
157Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Politica.xml
158Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Salute__Figli__e__Genitori.xml
159Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Salute__Sportello__Cancro.xml
160Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Elezioni.xml
161
162#CORRIERE-452 2018-10-08
163
164Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Tecnologia.xml
165Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Offerte__recensioni.xml
166Sitemap: https://www.corriere.it/dynamic-feed/rss/section/Dataroom.xml
167Sitemap: https://www.corriere.it/rss/sitemap/Motori.xml
168Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Spettacoli.xml
169Sitemap: https://vivimilano.corriere.it/sitemap_index.xml
170Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Cook.xml
171Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Tecnologia.xml
172Sitemap: https://www.corriere.it/oroscopo/sitemap.xml
173Sitemap: https://www.corriere.it/elezioni/sitemap/sitemap.xml
174Sitemap: https://www.corriere.it/sport/risultati-live/sitemap.xml
175Sitemap: https://www.corriere.it/salute/il-medico-risponde/sitemap.xml
176Sitemap: https://www.corriere.it/salute/il-medico-risponde/news/sitemap.xml
177Sitemap: https://www.corriere.it/rss/sitemap/lettere-al-direttore.xml
178Sitemap: https://www.corriere.it/rss/sitemap/lo-dico-al-corriere.xml
179Sitemap: https://www.corriere.it/rss/sitemap/Cook-Last.xml
180Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Scuola.xml
181Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Animali.xml
182Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Opinioni.xml
183Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Caffe-gramellini.xml
184Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Ultimo-banco.xml
185Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Letti-da-rifarei.xml
186Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Piccole-dosi.xml
187Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/L-angolo.xml
188Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Padiglione-italia.xml
189Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Facce-nuove.xml
190Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Ritorno-in-solferino.xml
191Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Oriente-occidente.xml
192Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Sette.xml
193Sitemap: https://www.corriere.it/economia/chiedi-esperto/sitemap.xml
194Sitemap: https://www.corriere.it/economia/chiedi-esperto/news/sitemap.xml
195Sitemap: https://www.corriere.it/studio/sitemap-studio.xml
196Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Moda.xml
197Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/BuoneNotizie.xml
198Sitemap: https://www.corriere.it/dynamic-feed/rss/section/lettere-al-direttore.xml
199Sitemap: https://www.corriere.it/dynamic-feed/rss/section/lo-dico-al-corriere.xml
200Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/lettere__al__direttore.xml
201Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/lo__dico__al__corriere.xml
202Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Frammenti-ferruccio-de-bortoli.xml
203Sitemap: https://www.corriere.it/dynamic-feed/rss/section/frammenti-di-ferruccio-de-bortoli.xml
204Sitemap: https://www.corriere.it/dynamic-sitemap/corriere/video/lezioni/sitemap-aggregation-index.xml
205Sitemap: https://www.corriere.it/dynamic-sitemap/corriere/le-lezioni-del-corriere/sitemap.xml
206Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Giochi.xml
207Sitemap: https://www.corriere.it/guida-autovelox-e-codice-della-strada/sitemap.xml
208Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Autovelox.xml
209Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Motori.xml
210Sitemap: https://borsa.corriere.it/sitemap.xml
211Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/nativeadv.xml
212Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/dalle_aziende.xml
213Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/150anni.xml
214Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Cultura.xml
215
216
217
218
219
220