NeuralCrawl

The Telegraph / robots.txt snapshot

← back to telegraph.co.uk · fetched 2026-08-26T14:01:29Z (2mo ago) · HTTP 200 · 5210 bytes · sha256 9e246848905e8b48 · raw

final URL: https://www.telegraph.co.uk/robots.txt

1# Production Robots.txt file
2
3User-agent: *
4#Special parameters
5Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.
6Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.
7Disallow: /*?mobile=true
8Disallow: /*?mobile=basic
9Disallow: /*?ModPagespeed=noscript
10Disallow: /*_jcr_content*
11Disallow: /*?source=rss
12Disallow: /puzzles/puzzle/*?source=
13
14# Internal Search
15Disallow: /search/
16Allow: /search/$
17
18# Special areas
19Disallow: /news/main.jhtml
20Disallow: /p/*/embed/
21Disallow: /secure/login/*
22Disallow: /content/telegraph/
23Disallow: /customer/secure/checkout/tesco/
24Disallow: /customer/secure/reset-password/
25Disallow: /telegraph/*
26Disallow: /news-app/*
27Disallow: /amp$
28Disallow: */application/*
29Disallow: */ixale/
30Disallow: /core/Content/
31Disallow: /promotions/emails/
32Disallow: /r/
33Disallow: /sponsored/travel/msc-cruises/
34Disallow: /travel/8711559/The-Telegraph-Travel-Awards-2011.html
35Disallow: /travel/hotel/e/*
36Disallow: /sponsored/staging/
37Disallow: /sponsored/business/lloyds-tsb-enterprise-awards/
38Disallow: /sponsored/earth/statoil/
39Disallow: /sponsored/motoring/alfa-romeo-cars/
40Disallow: /sponsored/motoring/vw-up/
41Disallow: /sponsored/property/all-saints-eastbourne/
42Disallow: /sponsored/supplement-portfolio/
43Disallow: /sponsored/travel/cunard-cruises/
44Disallow: /sponsored/travel/cruise-holidays/
45Disallow: /sponsored/travel/macau/macaumap/
46Disallow: /sponsored/travel/telegraph-cottages/
47Disallow: /sponsored/finance/spread-betting/
48Disallow: /sponsored/finance/retirement-annuity/
49Disallow: /sponsored/travel/hidden-britain/
50Disallow: /sponsored/business/sme-business-essentials/
51Disallow: /sponsored/in-the-know/london-cultural-attractions
52Disallow: /sponsored/in-the-know/london-dining
53Disallow: /sponsored/in-the-know/london-entertainment
54Disallow: /sponsored/in-the-know/london-lifestyle
55Disallow: /sponsored/in-the-know/london-nightlife
56Disallow: /sponsored/in-the-know/london-shopping
57Disallow: /sponsored/in-the-know/london-sport-activities
58Disallow: /sponsored/in-the-know/london-transport-accommodation
59Disallow: /sponsored/in-the-know/london-video-guides
60Disallow: /sponsored/motoring/suzuki-motorbikes/
61Disallow: /sponsored/technology/cool-list/
62Disallow: /travel/hotels/hotel-finder/
63Disallow: /podcasts-more/
64Disallow: /secure/register/
65Allow: /travel/hotels/hotel-finder/$
66Disallow: /martech/js/
67Disallow: /martech/css/
68Disallow: /martech-content/
69Disallow: /bin/telegraph/recombee-config
70Disallow: /*&p=
71Disallow: /customer/subscription/*?
72Disallow: /edge-redirection/
73
74#Bots which make unnecessary bot traffic
75
76User-Agent: endeca
77Disallow: /archive/
78Disallow: /search/*
79
80User-agent: AI2Bot
81Disallow: /
82
83User-agent: Ai2Bot-Dolma
84Disallow: /
85
86User-agent: Amazonbot
87Disallow: /
88
89User-agent: anthropic-ai
90Disallow: /
91
92User-agent: Applebot-Extended
93Disallow: /
94
95User-agent: bedrockbot
96Disallow: /
97
98User-agent: Bytespider
99Disallow: /
100
101User-agent: CCBot
102Disallow: /
103
104User-agent: ChatGLM-Spider
105Disallow: /
106
107User-agent: ChatGPT-User
108Disallow: /
109
110User-agent: Claude-SearchBot
111Disallow: /
112
113User-agent: Claude-User
114Disallow: /
115
116User-agent: Claude-Web
117Disallow: /
118
119User-agent: ClaudeBot
120Disallow: /
121
122User-agent: cohere-ai
123Disallow: /
124
125User-agent: Cotoyogi
126Disallow: /
127
128User-agent: DeepSeekBot
129Disallow: /
130
131User-agent: Diffbot
132Disallow: /
133
134User-agent: DuckAssistBot
135Disallow: /
136
137User-agent: FacebookBot
138Disallow: /
139
140User-agent: FriendlyCrawler
141Disallow: /
142
143User-agent: Google-CloudVertexBot
144Disallow: /
145
146User-agent: Google-Extended
147Allow: /
148
149User-agent: GoogleOther
150Disallow: /
151
152User-agent: GoogleOther-Image
153Disallow: /
154
155User-agent: GoogleOther-Video
156Disallow: /
157
158User-agent: GPTBot
159Disallow: /
160
161User-agent: Grok
162Disallow: /
163
164User-agent: iaskspider/2.0
165Disallow: /
166
167User-agent: ICC-Crawler
168Disallow: /
169
170User-agent: ImagesiftBot
171Disallow: /
172
173User-agent: img2dataset
174Disallow: /
175
176User-agent: ISSCyberRiskCrawler
177Disallow: /
178
179User-agent: Kangaroo Bot
180Disallow: /
181
182User-agent: KunatoCrawler
183Disallow: /
184
185User-agent: Meltwater
186Disallow: /
187
188User-agent: Meta-ExternalAgent
189Disallow: /
190
191User-agent: Meta-ExternalFetcher
192Disallow: /
193
194User-agent: MistralAI-User
195Disallow: /
196
197User-agent: OAI-Operator
198Disallow: /
199
200User-agent: OAI-SearchBot
201Disallow: /
202
203User-agent: omgili
204Disallow: /
205
206User-agent: omgilibot
207Disallow: /
208
209User-agent: PanguBot
210Disallow: /
211
212User-agent: Perplexity-User
213Disallow: /
214
215User-agent: PerplexityBot
216Disallow: /
217
218User-agent: PetalBot
219Disallow: /
220
221User-agent: QualifiedBot
222Disallow: /
223
224User-agent: Scrapy
225Disallow: /
226
227User-agent: Seekr
228Disallow: /
229
230User-agent: Sidetrade indexer bot
231Disallow: /
232
233User-agent: TikTokSpider
234Disallow: /
235
236User-agent: Timpibot
237Disallow: /
238
239User-agent: VelenPublicWebCrawler
240Disallow: /
241
242User-agent: WARDBot
243Disallow: /
244
245User-agent: Webzio-Extended
246Disallow: /
247
248User-agent: WRTNBot
249Disallow: /
250
251
252#Sitemap URLs
253Sitemap: https://www.telegraph.co.uk/sitemap.xml
254Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml
255Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml
256Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml
257Sitemap: https://www.telegraph.co.uk/vouchercodes/sitemap.xml
258Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml
259