NeuralCrawl

Corriere della Sera / robots.txt snapshot

← back to corriere.it · fetched 2026-06-26T04:48:20Z (1mo ago) · HTTP 200 · 9544 bytes · sha256 972ad4d06b963924 · raw

final URL: https://www.corriere.it/robots.txt

1# Inizio Bot Exclusion Protocol
2User-agent: GPTBot
3Disallow: /
4User-agent: CCBot
5Disallow: /
6User-agent: anthropic-ai
7Disallow: /
8User-agent: AmazonAdBot
9Allow: /
10User-agent: Google-CloudVertexBot
11Disallow: /
12User-agent: AppleBot
13Disallow: /
14User-agent: Google-Extended
15Disallow: /
16User-agent: Applebot-Extended
17Disallow: /
18User-agent: ClaudeBot
19Disallow: /
20User-agent: Amazonbot
21Disallow: /
22User-agent: Meta-ExternalAgent
23Disallow: /
24User-agent: PetalBot
25Disallow: /
26User-agent: Yandex
27Disallow: / # prohibits crawling for the entire site
28User-agent: Bytespider
29Disallow: /
30User-agent: Scrapy
31Disallow: /
32User-agent: Diffbot-User
33Disallow: /
34User-agent: YouBot
35Disallow: /
36User-agent: Timpibot
37Disallow: /
38# Fine Bot Exclusion Protocol
39
40User-agent: *
41Disallow: *{{data.c.link_offerId}}
42
43
44Disallow: *(none)
45Disallow: *?RedirectUrl
46Disallow: /privacy.shtml
47Disallow: /abbonati.Suspended/
48Disallow: /corcommunitynew/accesso/LogOut.do
49Disallow: /indexMethodeProve.shtml
50Disallow: /apw.js
51Disallow: /salute/cardiologia/sintomi-infartosintomi-infarto
52Disallow: /salute/cardiologia/sintomi-infartoheart-attack-symptoms
53Disallow: /salute/cardiologia/sintomi-infartosíntomas-ataque-cardíaco
54Disallow: /salute/cardiologia/sintomi-infartosymptomes-infarctus
55Disallow: /salute/cardiologia/sintomi-infartoсимптомы-инфаркта
56Disallow: /salute/cardiologia/sintomi-infarto心肌梗塞症状
57Disallow: /salute/cardiologia/sintomi-infartoطب القلب
58Disallow: /salute/cardiologia/sintomi-infartosimptome-de-atac-de-cord
59Disallow: /salute/sportello_cancro/jsp/*
60Disallow: /corrierecalcio/app/formazione/
61Disallow: /preview/
62Disallow: /dwc-xalok/
63Disallow: /ssi/
64Disallow: /tools/
65Disallow: /linked_webroots/
66Disallow: /english/includes_methode/
67Disallow: /includes_methode/
68Disallow: /community-corriere/
69Disallow: /widget/
70Disallow: /dwc/
71Disallow: /digital-edition/
72Disallow: /correlati/
73Disallow: /cinema/icortissimi-ended/
74Disallow: /ZCestino/
75Disallow: /prova/
76Disallow: /test/
77Disallow: /landing/wrapped-2023/
78Disallow: /test_mf/
79Disallow: /piano-511/
80Disallow: /b2b/
81Disallow: /_template/
82Disallow: /_docs/
83Disallow: /_temp/
84Disallow: /cf/
85Disallow: /servizi/
86Disallow: /salute/dizionario/dizionarioSearch.jsp
87Disallow: /sportello-cancro/dizionario/
88Disallow: /dilatua/proRecensione.do
89Disallow: /newsletter/nl_economia.html
90# vecchia ultima ora
91Disallow: /ultima_ora/
92#Dizionario della Salute
93Disallow: /salute/dizionario/dizionarioSearch.jsp
94Disallow: /salute/dizionario/pages/results.jsp
95Disallow: /cronache/speciali/2010/maturita/test-universita.shtml
96Disallow: /smallApp/tellafriend/
97Disallow: /*_print.html$
98Disallow: /*.xml
99Disallow: /account/
100Allow: /account/login$
101Disallow: /account/login?
102Disallow: /gallery/*/vuoto.shtml
103Disallow: *pwcmp=
104Disallow: *reader_id=
105Disallow: /cruciverba_old/
106Allow: /salute/il-medico-risponde/sitemap.xml
107Allow: /salute/il-medico-risponde/news/sitemap.xml
108Allow: /salute/sitemap-dizionario-corriere-salute.xml
109Allow: /dynamic-sitemap/sitemap-last-100/*.xml
110Allow: /dynamic-feed/*.xml
111Allow: /cook*sitemap*.xml
112Allow: /guida-autovelox-e-codice-della-strada/sitemap.xml
113Allow: /radio/$
114Allow: /radio/index.shtml
115Allow: /servizi/$
116Allow: /ultima_ora/agrnews.jsp
117Allow: /showcase/*.xml
118Allow: /economia/chiedi-esperto/sitemap.xml
119Allow: /economia/chiedi-esperto/news/sitemap.xml
120Allow: /sette/speciale-presidenziali-usa-2024-harris-contro-trump/rss/*.xml
121Allow: /dynamic-sitemap/corriere/video/lezioni*sitemap*.xml
122
123
124# item
125Disallow: /corrierepedia
126Disallow: /firme/pierluigi-battista_80
127Disallow: /cook/ricette/ricerca
128Disallow: /CorriereAppNews/
129Disallow: /corrierepoint/
130Disallow: /index_mobile_tst_hp_mob_b.shtml
131Disallow: /*app_v2
132Disallow: /*app_v1
133Allow: /sitemap*.xml
134# Allow: /notizie-ultima-ora/rss_col.xml
135Allow: /rss/*.xml
136Allow: /salute/sitemap-dizionario.xml
137Allow: /rss/ultimora.xml
138# Allow: /rss/homepage.xml
139Allow: /feed-hp/homepage-restyle-2025.xml
140# Allow: /notizie-ultima-ora/sitemap-news.xml
141#richiesta da Ruggiero BG27112011
142Allow: /editorspicks/
143Allow: /oroscopo/sitemap.xml
144Allow: /sport/risultati-live/sitemap.xml
145Allow: /elezioni/*.xml
146Disallow: /*?q=
147Disallow: /*?f=
148Disallow: /*?s=
149Disallow: /*?searchType=
150Disallow: /*articleurl=
151Disallow: /economia/quiz/
152Disallow: /offerta-speciale-limitata/
153Disallow: /*?*rid=*&*destination=*
154Disallow: /*?*destination=*&*rid=*
155Disallow: /nazionale-2025/
156
157Sitemap: https://www.corriere.it/feed-hp/homepage-restyle-2025.xml
158Sitemap: https://www.corriere.it/salute/sitemap-dizionario-corriere-salute.xml
159Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/video/Corriere.xml
160Sitemap: https://www.corriere.it/rss/sitemap_v2.xml
161Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Economia.xml
162Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Salute.xml
163Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Scienze.xml
164Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Interni.xml
165Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Esteri.xml
166Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Sport.xml
167Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Politica.xml
168Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Salute__Figli__e__Genitori.xml
169Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Salute__Sportello__Cancro.xml
170Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Elezioni.xml
171
172#CORRIERE-452 2018-10-08
173
174Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Tecnologia.xml
175Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Offerte__recensioni.xml
176Sitemap: https://www.corriere.it/dynamic-feed/rss/section/Dataroom.xml
177Sitemap: https://www.corriere.it/rss/sitemap/Motori.xml
178Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Spettacoli.xml
179Sitemap: https://vivimilano.corriere.it/sitemap_index.xml
180Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Cook.xml
181Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Tecnologia.xml
182Sitemap: https://www.corriere.it/oroscopo/sitemap.xml
183Sitemap: https://www.corriere.it/elezioni/sitemap/sitemap.xml
184Sitemap: https://www.corriere.it/sport/risultati-live/sitemap.xml
185Sitemap: https://www.corriere.it/salute/il-medico-risponde/sitemap.xml
186Sitemap: https://www.corriere.it/salute/il-medico-risponde/news/sitemap.xml
187Sitemap: https://www.corriere.it/rss/sitemap/lettere-al-direttore.xml
188Sitemap: https://www.corriere.it/rss/sitemap/lo-dico-al-corriere.xml
189Sitemap: https://www.corriere.it/rss/sitemap/Cook-Last.xml
190Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Scuola.xml
191Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Animali.xml
192Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Opinioni.xml
193Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Caffe-gramellini.xml
194Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Ultimo-banco.xml
195Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Letti-da-rifarei.xml
196Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Piccole-dosi.xml
197Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/L-angolo.xml
198Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Padiglione-italia.xml
199Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Facce-nuove.xml
200Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Ritorno-in-solferino.xml
201Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Oriente-occidente.xml
202Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Sette.xml
203Sitemap: https://www.corriere.it/economia/chiedi-esperto/sitemap.xml
204Sitemap: https://www.corriere.it/economia/chiedi-esperto/news/sitemap.xml
205Sitemap: https://www.corriere.it/studio/sitemap-studio.xml
206Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Moda.xml
207Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/BuoneNotizie.xml
208Sitemap: https://www.corriere.it/dynamic-feed/rss/section/lettere-al-direttore.xml
209Sitemap: https://www.corriere.it/dynamic-feed/rss/section/lo-dico-al-corriere.xml
210Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/lettere__al__direttore.xml
211Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/lo__dico__al__corriere.xml
212Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Frammenti-ferruccio-de-bortoli.xml
213Sitemap: https://www.corriere.it/dynamic-feed/rss/section/frammenti-di-ferruccio-de-bortoli.xml
214Sitemap: https://www.corriere.it/dynamic-sitemap/corriere/video/lezioni/sitemap-aggregation-index.xml
215Sitemap: https://www.corriere.it/dynamic-sitemap/corriere/le-lezioni-del-corriere/sitemap.xml
216Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Giochi.xml
217Sitemap: https://www.corriere.it/guida-autovelox-e-codice-della-strada/sitemap.xml
218Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Autovelox.xml
219Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Motori.xml
220Sitemap: https://borsa.corriere.it/sitemap.xml
221Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/nativeadv.xml
222Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/dalle_aziende.xml
223Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/150anni.xml
224Sitemap: https://www.corriere.it/dynamic-sitemap/sitemap-last-100/Cultura.xml
225
226
227
228
229
230