NeuralCrawl

The Telegraph / robots.txt snapshot

← back to telegraph.co.uk · fetched 2026-10-22T21:05:52Z (15d ago) · HTTP 200 · 5836 bytes · sha256 2aff8d5f0e77abba · raw

final URL: https://www.telegraph.co.uk/robots.txt

1# Production Robots.txt file
2
3User-agent: *
4User-agent: Google-Extended
5# Special parameters
6Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.*.js
7Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.*.js
8Disallow: /*?mobile=true
9Disallow: /*?mobile=basic
10Disallow: /*?ModPagespeed=noscript
11Disallow: /*_jcr_content*
12Disallow: /*?source=rss
13Disallow: /puzzles/puzzle/*?source=
14
15# Internal Search
16Disallow: /search/
17Allow: /search/$
18
19# Special areas
20Disallow: /news/main.jhtml
21Disallow: /p/*/embed/
22Disallow: /secure/login/*
23Disallow: /content/telegraph/
24Disallow: /customer/secure/checkout/tesco/
25Disallow: /customer/secure/reset-password/
26Disallow: /telegraph/*
27Disallow: /news-app/*
28Disallow: /amp$
29Disallow: /application/*
30Disallow: /ixale/
31Disallow: /core/Content/
32Disallow: /r/
33Disallow: /travel/hotel/e/*
34Disallow: /travel/hotels/hotel-finder/
35Disallow: /podcasts-more/
36Disallow: /secure/register/
37Allow: /travel/hotels/hotel-finder/$
38Disallow: /martech/js/
39Disallow: /martech/css/
40Disallow: /martech-content/
41Disallow: /bin/telegraph/recombee-config
42Disallow: /*&p=
43Disallow: /customer/subscription/*?
44Disallow: /edge-redirection/
45
46
47# Bots which make unnecessary bot traffic
48User-agent: endeca
49Disallow: /archive/
50Disallow: /search/*
51
52User-agent: AI2Bot
53User-agent: Ai2Bot-Dolma
54User-agent: Amazonbot
55User-agent: anthropic-ai
56User-agent: Applebot-Extended
57User-agent: bedrockbot
58User-agent: Bytespider
59User-agent: CCBot
60User-agent: ChatGLM-Spider
61User-agent: ClaudeBot
62User-agent: Claude-Web
63User-agent: cohere-ai
64User-agent: Cotoyogi
65User-agent: DeepSeekBot
66User-agent: Diffbot
67User-agent: FacebookBot
68User-agent: FriendlyCrawler
69User-agent: Google-CloudVertexBot
70User-agent: GoogleOther
71User-agent: GoogleOther-Image
72User-agent: GoogleOther-Video
73User-agent: GPTBot
74User-agent: Grok
75User-agent: iaskspider/2.0
76User-agent: ICC-Crawler
77User-agent: ImagesiftBot
78User-agent: img2dataset
79User-agent: ISSCyberRiskCrawler
80User-agent: Kangaroo Bot
81User-agent: KunatoCrawler
82User-agent: Meltwater
83User-agent: Meta-ExternalAgent
84User-agent: omgili
85User-agent: omgilibot
86User-agent: PanguBot
87User-agent: PetalBot
88User-agent: QualifiedBot
89User-agent: Scrapy
90User-agent: Seekr
91User-agent: Sidetrade indexer bot
92User-agent: TikTokSpider
93User-agent: Timpibot
94User-agent: VelenPublicWebCrawler
95User-agent: WARDBot
96User-agent: Webzio-Extended
97User-agent: WRTNBot
98User-agent: Claude-SearchBot
99User-agent: DuckAssistBot
100User-agent: OAI-SearchBot
101User-agent: PerplexityBot
102User-agent: ChatGPT-User
103User-agent: Claude-User
104User-agent: Meta-ExternalFetcher
105User-agent: MistralAI-User
106User-agent: OAI-Operator
107User-agent: Perplexity-User
108Disallow: /
109Allow: /travel/explore-dubai/
110Allow: /travel/luxury-stays/
111Allow: /business/uk-business-growth/
112Allow: /ads.txt
113Allow: /bah-avis/usa-sports-road-trip-map
114Allow: /betting/
115Allow: /british-airways-holidays/cyprus-hotel-map
116Allow: /business/ai-era-job-skills/
117Allow: /business/ai-workflow-transformation/
118Allow: /business/future-hr-technology/
119Allow: /business/golf-dp-world-tour/
120Allow: /business/search-marketing/
121Allow: /business/waste-management/
122Allow: /deals/
123Allow: /environment/woodland-action/
124Allow: /fashion/joseph-ribkoff-autumn-collection/
125Allow: /financial-services/
126Allow: /food-and-drink/m-and-s-seasonal-favourites/
127Allow: /food-and-drink/m-and-s-seasonal-favourites/organic-range/
128Allow: /food-and-drink/maldon-salt-perfect-pyramid-challenge/
129Allow: /food-and-drink/spanish-dining-uk/
130Allow: /football/global-football-industry-awards/
131Allow: /health-fitness/how-to-choose-private-hospital-hca-healthcare/
132Allow: /interiors/next-home-autumn-interiors-trends/
133Allow: /junior-golf/
134Allow: /lifestyle-solutions/
135Allow: /lifestyle/apl-media/
136Allow: /branded-content/
137Allow: /money/investing/fc-investment-trust/long-term-investing/
138Allow: /branded-content/
139Allow: /rugby-union/grassroots-rugby-heroes/
140Allow: /rugby-union/levelling-the-playing-field/
141Allow: /sailgp/inside-f50-catamaran
142Allow: /seabourn/explore-the-mediterranean
143Allow: /sport/sailgp-championship/
144Allow: /travel/active-family-holidays/
145Allow: /travel/away-holidays/
146Allow: /travel/barbados-holiday-guide/
147Allow: /travel/british-airways-american-express/
148Allow: /travel/canadian-affair-holidays/rocky-mountaineer-train-route/
149Allow: /travel/croatia-holidays/
150Allow: /travel/cruises/holland-america-line-luxury-holidays/
151Allow: /travel/cruises/seabourn-luxury-cruises/
152Allow: /travel/discover-america/
153Allow: /travel/exclusiveoffers/
154Allow: /branded-content/
155Allow: /travel/florida-guide/
156Allow: /travel/fly-drive-holidays/
157Allow: /travel/grenada-guide/
158Allow: /travel/icons-of-the-sky/
159Allow: /travel/indian-ocean-escapes/
160Allow: /travel/indulge-in-ireland/
161Allow: /travel/jamaica-holiday-guide/
162Allow: /travel/japan-ninja-heritage/
163Allow: /travel/luxury-all-inclusive-resorts/affordable-caribbean-hotels/
164Allow: /travel/luxury-holidays/
165Allow: /travel/luxury-hotel-getaways/
166Allow: /travel/luxury-island-escapes/
167Allow: /travel/maldives-holiday-guide/
168Allow: /travel/ski-resorts-for-everyone/
169Allow: /travel/spark/best-of-switzerland/
170Allow: /travel/tailor-made-tours/
171Allow: /travel/tenerife-holidays/
172Allow: /travel/tips-for-visiting-spain/
173Allow: /travel/travel-counsellors/book-a-cruise-with-travel-counsellors/
174Allow: /travel/travel-solutions/
175Allow: /travel/visit-cayman-islands/
176Allow: /travel/why-choose-malta/
177Allow: /trusted-partners/
178Allow: /brand-usa
179Allow: /travel/visit-aarhus/
180Allow: /bespoke-content/
181
182
183
184# Sitemap URLs
185Sitemap: https://www.telegraph.co.uk/sitemap.xml
186Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml
187Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml
188Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml
189Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml
190