NeuralCrawl

The Telegraph / robots.txt snapshot

← back to telegraph.co.uk · fetched 2026-11-03T13:00:12Z (3d ago) · HTTP 200 · 3231 bytes · sha256 a595536ebd86f17e · raw

final URL: https://www.telegraph.co.uk/robots.txt

1# Production Robots.txt file
2
3User-agent: *
4User-agent: Google-Extended
5# Special parameters
6Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.*.js
7Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.*.js
8Disallow: /*?mobile=true
9Disallow: /*?mobile=basic
10Disallow: /*?ModPagespeed=noscript
11Disallow: /*_jcr_content*
12Disallow: /*?source=rss
13Disallow: /puzzles/puzzle/*?source=
14
15# Internal Search
16Disallow: /search/
17Allow: /search/$
18
19# Special areas
20Disallow: /news/main.jhtml
21Disallow: /p/*/embed/
22Disallow: /secure/login/*
23Disallow: /content/telegraph/
24Disallow: /customer/secure/checkout/tesco/
25Disallow: /customer/secure/reset-password/
26Disallow: /telegraph/*
27Disallow: /news-app/*
28Disallow: /amp$
29Disallow: /application/*
30Disallow: /ixale/
31Disallow: /core/Content/
32Disallow: /promotions/emails/
33Disallow: /r/
34Disallow: /sponsored/travel/msc-cruises/
35Disallow: /travel/hotel/e/*
36Disallow: /travel/hotels/hotel-finder/
37Disallow: /podcasts-more/
38Disallow: /secure/register/
39Allow: /travel/hotels/hotel-finder/$
40Disallow: /martech/js/
41Disallow: /martech/css/
42Disallow: /martech-content/
43Disallow: /bin/telegraph/recombee-config
44Disallow: /*&p=
45Disallow: /customer/subscription/*?
46Disallow: /edge-redirection/
47Allow: /
48
49
50# Bots which make unnecessary bot traffic
51User-agent: endeca
52Disallow: /archive/
53Disallow: /search/*
54
55User-agent: AI2Bot
56User-agent: Ai2Bot-Dolma
57User-agent: Amazonbot
58User-agent: anthropic-ai
59User-agent: Applebot-Extended
60User-agent: bedrockbot
61User-agent: Bytespider
62User-agent: CCBot
63User-agent: ChatGLM-Spider
64User-agent: ClaudeBot
65User-agent: Claude-Web
66User-agent: cohere-ai
67User-agent: Cotoyogi
68User-agent: DeepSeekBot
69User-agent: Diffbot
70User-agent: FacebookBot
71User-agent: FriendlyCrawler
72User-agent: Google-CloudVertexBot
73User-agent: GoogleOther
74User-agent: GoogleOther-Image
75User-agent: GoogleOther-Video
76User-agent: GPTBot
77User-agent: Grok
78User-agent: iaskspider/2.0
79User-agent: ICC-Crawler
80User-agent: ImagesiftBot
81User-agent: img2dataset
82User-agent: ISSCyberRiskCrawler
83User-agent: Kangaroo Bot
84User-agent: KunatoCrawler
85User-agent: Meltwater
86User-agent: Meta-ExternalAgent
87User-agent: omgili
88User-agent: omgilibot
89User-agent: PanguBot
90User-agent: PetalBot
91User-agent: QualifiedBot
92User-agent: Scrapy
93User-agent: Seekr
94User-agent: Sidetrade indexer bot
95User-agent: TikTokSpider
96User-agent: Timpibot
97User-agent: VelenPublicWebCrawler
98User-agent: WARDBot
99User-agent: Webzio-Extended
100User-agent: WRTNBot
101User-agent: Claude-SearchBot
102User-agent: DuckAssistBot
103User-agent: OAI-SearchBot
104User-agent: PerplexityBot
105User-agent: ChatGPT-User
106User-agent: Claude-User
107User-agent: Meta-ExternalFetcher
108User-agent: MistralAI-User
109User-agent: OAI-Operator
110User-agent: Perplexity-User
111Disallow: /
112Allow: /travel/explore-dubai/
113Allow: /travel/luxury-stays/
114Allow: /business/uk-business-growth/
115Allow: /ads.txt
116
117
118# Sitemap URLs
119Sitemap: https://www.telegraph.co.uk/sitemap.xml
120Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml
121Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml
122Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml
123Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml