NeuralCrawl

The Telegraph / robots.txt snapshot

← back to telegraph.co.uk · fetched 2026-10-08T10:00:34Z (30d ago) · HTTP 200 · 3228 bytes · sha256 a32d600f469bb0fd · raw

final URL: https://www.telegraph.co.uk/robots.txt

1# Production Robots.txt file
2
3User-agent: *
4User-agent: Google-Extended
5# Special parameters
6Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.*.js
7Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.*.js
8Disallow: /*?mobile=true
9Disallow: /*?mobile=basic
10Disallow: /*?ModPagespeed=noscript
11Disallow: /*_jcr_content*
12Disallow: /*?source=rss
13Disallow: /puzzles/puzzle/*?source=
14
15# Internal Search
16Disallow: /search/
17Allow: /search/$
18
19# Special areas
20Disallow: /news/main.jhtml
21Disallow: /p/*/embed/
22Disallow: /secure/login/*
23Disallow: /content/telegraph/
24Disallow: /customer/secure/checkout/tesco/
25Disallow: /customer/secure/reset-password/
26Disallow: /telegraph/*
27Disallow: /news-app/*
28Disallow: /amp$
29Disallow: /application/*
30Disallow: /ixale/
31Disallow: /core/Content/
32Disallow: /promotions/emails/
33Disallow: /r/
34Disallow: /sponsored/travel/msc-cruises/
35Disallow: /travel/hotel/e/*
36Disallow: /travel/hotels/hotel-finder/
37Disallow: /podcasts-more/
38Disallow: /secure/register/
39Allow: /travel/hotels/hotel-finder/$
40Disallow: /martech/js/
41Disallow: /martech/css/
42Disallow: /martech-content/
43Disallow: /bin/telegraph/recombee-config
44Disallow: /*&p=
45Disallow: /customer/subscription/*?
46Disallow: /edge-redirection/
47
48
49# Bots which make unnecessary bot traffic
50User-agent: endeca
51Disallow: /archive/
52Disallow: /search/*
53
54User-agent: AI2Bot
55User-agent: Ai2Bot-Dolma
56User-agent: Amazonbot
57User-agent: anthropic-ai
58User-agent: Applebot-Extended
59User-agent: bedrockbot
60User-agent: Bytespider
61User-agent: CCBot
62User-agent: ChatGLM-Spider
63User-agent: ClaudeBot
64User-agent: Claude-Web
65User-agent: cohere-ai
66User-agent: Cotoyogi
67User-agent: DeepSeekBot
68User-agent: Diffbot
69User-agent: FacebookBot
70User-agent: FriendlyCrawler
71User-agent: Google-CloudVertexBot
72User-agent: GoogleOther
73User-agent: GoogleOther-Image
74User-agent: GoogleOther-Video
75User-agent: GPTBot
76User-agent: Grok
77User-agent: iaskspider/2.0
78User-agent: ICC-Crawler
79User-agent: ImagesiftBot
80User-agent: img2dataset
81User-agent: ISSCyberRiskCrawler
82User-agent: Kangaroo Bot
83User-agent: KunatoCrawler
84User-agent: Meltwater
85User-agent: Meta-ExternalAgent
86User-agent: omgili
87User-agent: omgilibot
88User-agent: PanguBot
89User-agent: PetalBot
90User-agent: QualifiedBot
91User-agent: Scrapy
92User-agent: Seekr
93User-agent: Sidetrade indexer bot
94User-agent: TikTokSpider
95User-agent: Timpibot
96User-agent: VelenPublicWebCrawler
97User-agent: WARDBot
98User-agent: Webzio-Extended
99User-agent: WRTNBot
100User-agent: Claude-SearchBot
101User-agent: DuckAssistBot
102User-agent: OAI-SearchBot
103User-agent: PerplexityBot
104User-agent: ChatGPT-User
105User-agent: Claude-User
106User-agent: Meta-ExternalFetcher
107User-agent: MistralAI-User
108User-agent: OAI-Operator
109User-agent: Perplexity-User
110Disallow: /
111Allow: /travel/explore-dubai/
112Allow: /travel/luxury-stays/
113Allow: /business/uk-business-growth/
114Allow: /ads.txt
115
116
117# Sitemap URLs
118Sitemap: https://www.telegraph.co.uk/sitemap.xml
119Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml
120Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml
121Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml
122Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml
123
124
125
126
127
128