NeuralCrawl

The Telegraph / robots.txt snapshot

← back to telegraph.co.uk · fetched 2026-06-20T01:10:30Z (13h ago) · HTTP 200 · 5184 bytes · sha256 d8bcad927f4642a5 · raw

final URL: https://www.telegraph.co.uk/robots.txt

1# Production Robots.txt file
2
3User-agent: *
4#Special parameters
5Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.
6Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.
7Disallow: /*?mobile=true
8Disallow: /*?mobile=basic
9Disallow: /*?ModPagespeed=noscript
10Disallow: /*_jcr_content*
11Disallow: /*?source=rss
12Disallow: /puzzles/puzzle/*?source=
13
14# Internal Search
15Disallow: /search/
16Allow: /search/$
17
18# Special areas
19Disallow: /news/main.jhtml
20Disallow: /p/*/embed/
21Disallow: /secure/login/*
22Disallow: /content/telegraph/
23Disallow: /customer/secure/checkout/tesco/
24Disallow: /customer/secure/reset-password/
25Disallow: /telegraph/*
26Disallow: /news-app/*
27Disallow: /amp$
28Disallow: */application/*
29Disallow: */ixale/
30Disallow: /core/Content/
31Disallow: /promotions/emails/
32Disallow: /r/
33Disallow: /sponsored/travel/msc-cruises/
34Disallow: /travel/8711559/The-Telegraph-Travel-Awards-2011.html
35Disallow: /travel/hotel/e/*
36Disallow: /sponsored/staging/
37Disallow: /sponsored/business/lloyds-tsb-enterprise-awards/
38Disallow: /sponsored/earth/statoil/
39Disallow: /sponsored/motoring/alfa-romeo-cars/
40Disallow: /sponsored/motoring/vw-up/
41Disallow: /sponsored/property/all-saints-eastbourne/
42Disallow: /sponsored/supplement-portfolio/
43Disallow: /sponsored/travel/cunard-cruises/
44Disallow: /sponsored/travel/cruise-holidays/
45Disallow: /sponsored/travel/macau/macaumap/
46Disallow: /sponsored/travel/telegraph-cottages/
47Disallow: /sponsored/finance/spread-betting/
48Disallow: /sponsored/finance/retirement-annuity/
49Disallow: /sponsored/travel/hidden-britain/
50Disallow: /sponsored/business/sme-business-essentials/
51Disallow: /sponsored/in-the-know/london-cultural-attractions
52Disallow: /sponsored/in-the-know/london-dining
53Disallow: /sponsored/in-the-know/london-entertainment
54Disallow: /sponsored/in-the-know/london-lifestyle
55Disallow: /sponsored/in-the-know/london-nightlife
56Disallow: /sponsored/in-the-know/london-shopping
57Disallow: /sponsored/in-the-know/london-sport-activities
58Disallow: /sponsored/in-the-know/london-transport-accommodation
59Disallow: /sponsored/in-the-know/london-video-guides
60Disallow: /sponsored/motoring/suzuki-motorbikes/
61Disallow: /sponsored/technology/cool-list/
62Disallow: /travel/hotels/hotel-finder/
63Disallow: /podcasts-more/
64Disallow: /secure/register/
65Allow: /travel/hotels/hotel-finder/$
66Disallow: /martech/js/
67Disallow: /martech/css/
68Disallow: /martech-content/
69Disallow: /bin/telegraph/recombee-config
70Disallow: /*&p=
71Disallow: /customer/subscription/*?
72
73#Bots which make unnecessary bot traffic
74
75User-Agent: endeca
76Disallow: /archive/
77Disallow: /search/*
78
79User-agent: AI2Bot
80Disallow: /
81
82User-agent: Ai2Bot-Dolma
83Disallow: /
84
85User-agent: Amazonbot
86Disallow: /
87
88User-agent: anthropic-ai
89Disallow: /
90
91User-agent: Applebot-Extended
92Disallow: /
93
94User-agent: bedrockbot
95Disallow: /
96
97User-agent: Bytespider
98Disallow: /
99
100User-agent: CCBot
101Disallow: /
102
103User-agent: ChatGLM-Spider
104Disallow: /
105
106User-agent: ChatGPT-User
107Disallow: /
108
109User-agent: Claude-SearchBot
110Disallow: /
111
112User-agent: Claude-User
113Disallow: /
114
115User-agent: Claude-Web
116Disallow: /
117
118User-agent: ClaudeBot
119Disallow: /
120
121User-agent: cohere-ai
122Disallow: /
123
124User-agent: Cotoyogi
125Disallow: /
126
127User-agent: DeepSeekBot
128Disallow: /
129
130User-agent: Diffbot
131Disallow: /
132
133User-agent: DuckAssistBot
134Disallow: /
135
136User-agent: FacebookBot
137Disallow: /
138
139User-agent: FriendlyCrawler
140Disallow: /
141
142User-agent: Google-CloudVertexBot
143Disallow: /
144
145User-agent: Google-Extended
146Disallow: /
147
148User-agent: GoogleOther
149Disallow: /
150
151User-agent: GoogleOther-Image
152Disallow: /
153
154User-agent: GoogleOther-Video
155Disallow: /
156
157User-agent: GPTBot
158Disallow: /
159
160User-agent: Grok
161Disallow: /
162
163User-agent: iaskspider/2.0
164Disallow: /
165
166User-agent: ICC-Crawler
167Disallow: /
168
169User-agent: ImagesiftBot
170Disallow: /
171
172User-agent: img2dataset
173Disallow: /
174
175User-agent: ISSCyberRiskCrawler
176Disallow: /
177
178User-agent: Kangaroo Bot
179Disallow: /
180
181User-agent: KunatoCrawler
182Disallow: /
183
184User-agent: Meltwater
185Disallow: /
186
187User-agent: Meta-ExternalAgent
188Disallow: /
189
190User-agent: Meta-ExternalFetcher
191Disallow: /
192
193User-agent: MistralAI-User
194Disallow: /
195
196User-agent: OAI-Operator
197Disallow: /
198
199User-agent: OAI-SearchBot
200Disallow: /
201
202User-agent: omgili
203Disallow: /
204
205User-agent: omgilibot
206Disallow: /
207
208User-agent: PanguBot
209Disallow: /
210
211User-agent: Perplexity-User
212Disallow: /
213
214User-agent: PerplexityBot
215Disallow: /
216
217User-agent: PetalBot
218Disallow: /
219
220User-agent: QualifiedBot
221Disallow: /
222
223User-agent: Scrapy
224Disallow: /
225
226User-agent: Seekr
227Disallow: /
228
229User-agent: Sidetrade indexer bot
230Disallow: /
231
232User-agent: TikTokSpider
233Disallow: /
234
235User-agent: Timpibot
236Disallow: /
237
238User-agent: VelenPublicWebCrawler
239Disallow: /
240
241User-agent: WARDBot
242Disallow: /
243
244User-agent: Webzio-Extended
245Disallow: /
246
247User-agent: WRTNBot
248Disallow: /
249
250
251#Sitemap URLs
252Sitemap: https://www.telegraph.co.uk/sitemap.xml
253Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml
254Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml
255Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml
256Sitemap: https://www.telegraph.co.uk/vouchercodes/sitemap.xml
257Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml
258