NeuralCrawl

The Telegraph / robots.txt snapshot

← back to telegraph.co.uk · fetched 2026-11-05T19:53:48Z (1d ago) · HTTP 200 · 5913 bytes · sha256 f5db91e3a85f52b4 · raw

final URL: https://www.telegraph.co.uk/robots.txt

1# Production Robots.txt file
2
3User-agent: *
4User-agent: Google-Extended
5# Special parameters
6Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.*.js
7Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.*.js
8Disallow: /*?mobile=true
9Disallow: /*?mobile=basic
10Disallow: /*?ModPagespeed=noscript
11Disallow: /*_jcr_content*
12Disallow: /*?source=rss
13Disallow: /puzzles/puzzle/*?source=
14
15# Internal Search
16Disallow: /search/
17Allow: /search/$
18
19# Special areas
20Disallow: /news/main.jhtml
21Disallow: /p/*/embed/
22Disallow: /secure/login/*
23Disallow: /content/telegraph/
24Disallow: /customer/secure/checkout/tesco/
25Disallow: /customer/secure/reset-password/
26Disallow: /telegraph/*
27Disallow: /news-app/*
28Disallow: /amp$
29Disallow: /application/*
30Disallow: /ixale/
31Disallow: /core/Content/
32Disallow: /promotions/emails/
33Disallow: /r/
34Disallow: /sponsored/travel/msc-cruises/
35Disallow: /travel/hotel/e/*
36Disallow: /travel/hotels/hotel-finder/
37Disallow: /podcasts-more/
38Disallow: /secure/register/
39Allow: /travel/hotels/hotel-finder/$
40Disallow: /martech/js/
41Disallow: /martech/css/
42Disallow: /martech-content/
43Disallow: /bin/telegraph/recombee-config
44Disallow: /*&p=
45Disallow: /customer/subscription/*?
46Disallow: /edge-redirection/
47Allow: /
48
49
50# Bots which make unnecessary bot traffic
51User-agent: endeca
52Disallow: /archive/
53Disallow: /search/*
54
55User-agent: AI2Bot
56User-agent: Ai2Bot-Dolma
57User-agent: Amazonbot
58User-agent: anthropic-ai
59User-agent: Applebot-Extended
60User-agent: bedrockbot
61User-agent: Bytespider
62User-agent: CCBot
63User-agent: ChatGLM-Spider
64User-agent: ClaudeBot
65User-agent: Claude-Web
66User-agent: cohere-ai
67User-agent: Cotoyogi
68User-agent: DeepSeekBot
69User-agent: Diffbot
70User-agent: FacebookBot
71User-agent: FriendlyCrawler
72User-agent: Google-CloudVertexBot
73User-agent: GoogleOther
74User-agent: GoogleOther-Image
75User-agent: GoogleOther-Video
76User-agent: GPTBot
77User-agent: Grok
78User-agent: iaskspider/2.0
79User-agent: ICC-Crawler
80User-agent: ImagesiftBot
81User-agent: img2dataset
82User-agent: ISSCyberRiskCrawler
83User-agent: Kangaroo Bot
84User-agent: KunatoCrawler
85User-agent: Meltwater
86User-agent: Meta-ExternalAgent
87User-agent: omgili
88User-agent: omgilibot
89User-agent: PanguBot
90User-agent: PetalBot
91User-agent: QualifiedBot
92User-agent: Scrapy
93User-agent: Seekr
94User-agent: Sidetrade indexer bot
95User-agent: TikTokSpider
96User-agent: Timpibot
97User-agent: VelenPublicWebCrawler
98User-agent: WARDBot
99User-agent: Webzio-Extended
100User-agent: WRTNBot
101User-agent: Claude-SearchBot
102User-agent: DuckAssistBot
103User-agent: OAI-SearchBot
104User-agent: PerplexityBot
105User-agent: ChatGPT-User
106User-agent: Claude-User
107User-agent: Meta-ExternalFetcher
108User-agent: MistralAI-User
109User-agent: OAI-Operator
110User-agent: Perplexity-User
111Disallow: /
112Allow: /travel/explore-dubai/
113Allow: /travel/luxury-stays/
114Allow: /business/uk-business-growth/
115Allow: /ads.txt
116Allow: /bah-avis/usa-sports-road-trip-map
117Allow: /betting/
118Allow: /british-airways-holidays/cyprus-hotel-map
119Allow: /business/ai-era-job-skills/
120Allow: /business/ai-workflow-transformation/
121Allow: /business/future-hr-technology/
122Allow: /business/golf-dp-world-tour/
123Allow: /business/search-marketing/
124Allow: /business/waste-management/
125Allow: /deals/
126Allow: /environment/woodland-action/
127Allow: /fashion/joseph-ribkoff-autumn-collection/
128Allow: /financial-services/
129Allow: /food-and-drink/m-and-s-seasonal-favourites/
130Allow: /food-and-drink/m-and-s-seasonal-favourites/organic-range/
131Allow: /food-and-drink/maldon-salt-perfect-pyramid-challenge/
132Allow: /food-and-drink/spanish-dining-uk/
133Allow: /football/global-football-industry-awards/
134Allow: /health-fitness/how-to-choose-private-hospital-hca-healthcare/
135Allow: /interiors/next-home-autumn-interiors-trends/
136Allow: /junior-golf/
137Allow: /lifestyle-solutions/
138Allow: /lifestyle/apl-media/
139Allow: /branded-content/
140Allow: /money/investing/fc-investment-trust/long-term-investing/
141Allow: /branded-content/
142Allow: /rugby-union/grassroots-rugby-heroes/
143Allow: /rugby-union/levelling-the-playing-field/
144Allow: /sailgp/inside-f50-catamaran
145Allow: /seabourn/explore-the-mediterranean
146Allow: /sport/sailgp-championship/
147Allow: /travel/active-family-holidays/
148Allow: /travel/away-holidays/
149Allow: /travel/barbados-holiday-guide/
150Allow: /travel/british-airways-american-express/
151Allow: /travel/canadian-affair-holidays/rocky-mountaineer-train-route/
152Allow: /travel/croatia-holidays/
153Allow: /travel/cruises/holland-america-line-luxury-holidays/
154Allow: /travel/cruises/seabourn-luxury-cruises/
155Allow: /travel/discover-america/
156Allow: /travel/exclusiveoffers/
157Allow: /branded-content/
158Allow: /travel/florida-guide/
159Allow: /travel/fly-drive-holidays/
160Allow: /travel/grenada-guide/
161Allow: /travel/icons-of-the-sky/
162Allow: /travel/indian-ocean-escapes/
163Allow: /travel/indulge-in-ireland/
164Allow: /travel/jamaica-holiday-guide/
165Allow: /travel/japan-ninja-heritage/
166Allow: /travel/luxury-all-inclusive-resorts/affordable-caribbean-hotels/
167Allow: /travel/luxury-holidays/
168Allow: /travel/luxury-hotel-getaways/
169Allow: /travel/luxury-island-escapes/
170Allow: /travel/maldives-holiday-guide/
171Allow: /travel/ski-resorts-for-everyone/
172Allow: /travel/spark/best-of-switzerland/
173Allow: /travel/tailor-made-tours/
174Allow: /travel/tenerife-holidays/
175Allow: /travel/tips-for-visiting-spain/
176Allow: /travel/travel-counsellors/book-a-cruise-with-travel-counsellors/
177Allow: /travel/travel-solutions/
178Allow: /travel/visit-cayman-islands/
179Allow: /travel/why-choose-malta/
180Allow: /trusted-partners/
181Allow: /brand-usa
182Allow: /travel/visit-aarhus/
183Allow: /bespoke-content/
184
185# Sitemap URLs
186Sitemap: https://www.telegraph.co.uk/sitemap.xml
187Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml
188Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml
189Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml
190Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml