The Telegraph / robots.txt
← back to telegraph.co.uk · change detected 1mo ago (2026-10-05T15:06:47Z) · modified +71 −205
What changed
- Added Disallow: /application/* for *
- Added Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.*.js for *
- Added Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.*.js for *
- Added Disallow: /ixale/ for *
- Removed Disallow: */application/* for *
- Removed Disallow: */ixale/ for *
- Removed Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh. for *
- Removed Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article. for *
- Removed Disallow: /sponsored/business/lloyds-tsb-enterprise-awards/ for *
- Removed Disallow: /sponsored/business/sme-business-essentials/ for *
- Removed Disallow: /sponsored/earth/statoil/ for *
- Removed Disallow: /sponsored/finance/retirement-annuity/ for *
- Removed Disallow: /sponsored/finance/spread-betting/ for *
- Removed Disallow: /sponsored/in-the-know/london-cultural-attractions for *
- Removed Disallow: /sponsored/in-the-know/london-dining for *
- Removed Disallow: /sponsored/in-the-know/london-entertainment for *
- Removed Disallow: /sponsored/in-the-know/london-lifestyle for *
- Removed Disallow: /sponsored/in-the-know/london-nightlife for *
- Removed Disallow: /sponsored/in-the-know/london-shopping for *
- Removed Disallow: /sponsored/in-the-know/london-sport-activities for *
- Removed Disallow: /sponsored/in-the-know/london-transport-accommodation for *
- Removed Disallow: /sponsored/in-the-know/london-video-guides for *
- Removed Disallow: /sponsored/motoring/alfa-romeo-cars/ for *
- Removed Disallow: /sponsored/motoring/suzuki-motorbikes/ for *
- Removed Disallow: /sponsored/motoring/vw-up/ for *
- Removed Disallow: /sponsored/property/all-saints-eastbourne/ for *
- Removed Disallow: /sponsored/staging/ for *
- Removed Disallow: /sponsored/supplement-portfolio/ for *
- Removed Disallow: /sponsored/technology/cool-list/ for *
- Removed Disallow: /sponsored/travel/cruise-holidays/ for *
- Removed Disallow: /sponsored/travel/cunard-cruises/ for *
- Removed Disallow: /sponsored/travel/hidden-britain/ for *
- Removed Disallow: /sponsored/travel/macau/macaumap/ for *
- Removed Disallow: /sponsored/travel/telegraph-cottages/ for *
- Removed Disallow: /travel/8711559/The-Telegraph-Travel-Awards-2011.html for *
- 🤖 AIAdded Allow: /ads.txt for AI2Bot
- 🤖 AIAdded Allow: /business/uk-business-growth/ for AI2Bot
- 🤖 AIAdded Allow: /travel/explore-dubai/ for AI2Bot
- 🤖 AIAdded Allow: /travel/luxury-stays/ for AI2Bot
- Added Allow: /ads.txt for Ai2Bot-Dolma
- Added Allow: /business/uk-business-growth/ for Ai2Bot-Dolma
- Added Allow: /travel/explore-dubai/ for Ai2Bot-Dolma
- Added Allow: /travel/luxury-stays/ for Ai2Bot-Dolma
- 🤖 AIAdded Allow: /ads.txt for Amazonbot
- 🤖 AIAdded Allow: /business/uk-business-growth/ for Amazonbot
- 🤖 AIAdded Allow: /travel/explore-dubai/ for Amazonbot
- 🤖 AIAdded Allow: /travel/luxury-stays/ for Amazonbot
- 🤖 AIAdded Allow: /ads.txt for anthropic-ai
- 🤖 AIAdded Allow: /business/uk-business-growth/ for anthropic-ai
- 🤖 AIAdded Allow: /travel/explore-dubai/ for anthropic-ai
- ... and 248 more change(s)
Diff old (2026-08-26T14:01:29Z) → new (2026-10-05T15:06:47Z)
| @@ -1,9 +1,10 @@ | ||
| 1 | 1 | # Production Robots.txt file |
| 2 | 2 | |
| 3 | 3 | User-agent: * |
| 4 | - #Special parameters | |
| 5 | - Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article. | |
| 6 | - Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh. | |
| 4 | + User-agent: Google-Extended | |
| 5 | + # Special parameters | |
| 6 | + Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/save-article.*.js | |
| 7 | + Disallow: /etc.clientlibs/settings/wcm/designs/telegraph/core/clientlibs/page-refresh.*.js | |
| 7 | 8 | Disallow: /*?mobile=true |
| 8 | 9 | Disallow: /*?mobile=basic |
| 9 | 10 | Disallow: /*?ModPagespeed=noscript |
| @@ -25,40 +26,13 @@ | ||
| 25 | 26 | Disallow: /telegraph/* |
| 26 | 27 | Disallow: /news-app/* |
| 27 | 28 | Disallow: /amp$ |
| 28 | - Disallow: */application/* | |
| 29 | - Disallow: */ixale/ | |
| 29 | + Disallow: /application/* | |
| 30 | + Disallow: /ixale/ | |
| 30 | 31 | Disallow: /core/Content/ |
| 31 | 32 | Disallow: /promotions/emails/ |
| 32 | 33 | Disallow: /r/ |
| 33 | 34 | Disallow: /sponsored/travel/msc-cruises/ |
| 34 | - Disallow: /travel/8711559/The-Telegraph-Travel-Awards-2011.html | |
| 35 | 35 | Disallow: /travel/hotel/e/* |
| 36 | - Disallow: /sponsored/staging/ | |
| 37 | - Disallow: /sponsored/business/lloyds-tsb-enterprise-awards/ | |
| 38 | - Disallow: /sponsored/earth/statoil/ | |
| 39 | - Disallow: /sponsored/motoring/alfa-romeo-cars/ | |
| 40 | - Disallow: /sponsored/motoring/vw-up/ | |
| 41 | - Disallow: /sponsored/property/all-saints-eastbourne/ | |
| 42 | - Disallow: /sponsored/supplement-portfolio/ | |
| 43 | - Disallow: /sponsored/travel/cunard-cruises/ | |
| 44 | - Disallow: /sponsored/travel/cruise-holidays/ | |
| 45 | - Disallow: /sponsored/travel/macau/macaumap/ | |
| 46 | - Disallow: /sponsored/travel/telegraph-cottages/ | |
| 47 | - Disallow: /sponsored/finance/spread-betting/ | |
| 48 | - Disallow: /sponsored/finance/retirement-annuity/ | |
| 49 | - Disallow: /sponsored/travel/hidden-britain/ | |
| 50 | - Disallow: /sponsored/business/sme-business-essentials/ | |
| 51 | - Disallow: /sponsored/in-the-know/london-cultural-attractions | |
| 52 | - Disallow: /sponsored/in-the-know/london-dining | |
| 53 | - Disallow: /sponsored/in-the-know/london-entertainment | |
| 54 | - Disallow: /sponsored/in-the-know/london-lifestyle | |
| 55 | - Disallow: /sponsored/in-the-know/london-nightlife | |
| 56 | - Disallow: /sponsored/in-the-know/london-shopping | |
| 57 | - Disallow: /sponsored/in-the-know/london-sport-activities | |
| 58 | - Disallow: /sponsored/in-the-know/london-transport-accommodation | |
| 59 | - Disallow: /sponsored/in-the-know/london-video-guides | |
| 60 | - Disallow: /sponsored/motoring/suzuki-motorbikes/ | |
| 61 | - Disallow: /sponsored/technology/cool-list/ | |
| 62 | 36 | Disallow: /travel/hotels/hotel-finder/ |
| 63 | 37 | Disallow: /podcasts-more/ |
| 64 | 38 | Disallow: /secure/register/ |
| @@ -71,189 +45,81 @@ | ||
| 71 | 45 | Disallow: /customer/subscription/*? |
| 72 | 46 | Disallow: /edge-redirection/ |
| 73 | 47 | |
| 74 | - #Bots which make unnecessary bot traffic | |
| 75 | 48 | |
| 76 | - User-Agent: endeca | |
| 49 | + # Bots which make unnecessary bot traffic | |
| 50 | + User-agent: endeca | |
| 77 | 51 | Disallow: /archive/ |
| 78 | 52 | Disallow: /search/* |
| 79 | 53 | |
| 80 | 54 | User-agent: AI2Bot |
| 55 | + User-agent: Ai2Bot-Dolma | |
| 56 | + User-agent: Amazonbot | |
| 57 | + User-agent: anthropic-ai | |
| 58 | + User-agent: Applebot-Extended | |
| 59 | + User-agent: bedrockbot | |
| 60 | + User-agent: Bytespider | |
| 61 | + User-agent: CCBot | |
| 62 | + User-agent: ChatGLM-Spider | |
| 63 | + User-agent: ClaudeBot | |
| 64 | + User-agent: Claude-Web | |
| 65 | + User-agent: cohere-ai | |
| 66 | + User-agent: Cotoyogi | |
| 67 | + User-agent: DeepSeekBot | |
| 68 | + User-agent: Diffbot | |
| 69 | + User-agent: FacebookBot | |
| 70 | + User-agent: FriendlyCrawler | |
| 71 | + User-agent: Google-CloudVertexBot | |
| 72 | + User-agent: GoogleOther | |
| 73 | + User-agent: GoogleOther-Image | |
| 74 | + User-agent: GoogleOther-Video | |
| 75 | + User-agent: GPTBot | |
| 76 | + User-agent: Grok | |
| 77 | + User-agent: iaskspider/2.0 | |
| 78 | + User-agent: ICC-Crawler | |
| 79 | + User-agent: ImagesiftBot | |
| 80 | + User-agent: img2dataset | |
| 81 | + User-agent: ISSCyberRiskCrawler | |
| 82 | + User-agent: Kangaroo Bot | |
| 83 | + User-agent: KunatoCrawler | |
| 84 | + User-agent: Meltwater | |
| 85 | + User-agent: Meta-ExternalAgent | |
| 86 | + User-agent: omgili | |
| 87 | + User-agent: omgilibot | |
| 88 | + User-agent: PanguBot | |
| 89 | + User-agent: PetalBot | |
| 90 | + User-agent: QualifiedBot | |
| 91 | + User-agent: Scrapy | |
| 92 | + User-agent: Seekr | |
| 93 | + User-agent: Sidetrade indexer bot | |
| 94 | + User-agent: TikTokSpider | |
| 95 | + User-agent: Timpibot | |
| 96 | + User-agent: VelenPublicWebCrawler | |
| 97 | + User-agent: WARDBot | |
| 98 | + User-agent: Webzio-Extended | |
| 99 | + User-agent: WRTNBot | |
| 100 | + User-agent: Claude-SearchBot | |
| 101 | + User-agent: DuckAssistBot | |
| 102 | + User-agent: OAI-SearchBot | |
| 103 | + User-agent: PerplexityBot | |
| 104 | + User-agent: ChatGPT-User | |
| 105 | + User-agent: Claude-User | |
| 106 | + User-agent: Meta-ExternalFetcher | |
| 107 | + User-agent: MistralAI-User | |
| 108 | + User-agent: OAI-Operator | |
| 109 | + User-agent: Perplexity-User | |
| 110 | + Allow: /travel/explore-dubai/ | |
| 111 | + Allow: /travel/luxury-stays/ | |
| 112 | + Allow: /business/uk-business-growth/ | |
| 113 | + Allow: /ads.txt | |
| 81 | 114 | Disallow: / |
| 82 | 115 | |
| 83 | - User-agent: Ai2Bot-Dolma | |
| 84 | - Disallow: / | |
| 85 | - | |
| 86 | - User-agent: Amazonbot | |
| 87 | - Disallow: / | |
| 88 | - | |
| 89 | - User-agent: anthropic-ai | |
| 90 | - Disallow: / | |
| 91 | - | |
| 92 | - User-agent: Applebot-Extended | |
| 93 | - Disallow: / | |
| 94 | - | |
| 95 | - User-agent: bedrockbot | |
| 96 | - Disallow: / | |
| 97 | - | |
| 98 | - User-agent: Bytespider | |
| 99 | - Disallow: / | |
| 100 | - | |
| 101 | - User-agent: CCBot | |
| 102 | - Disallow: / | |
| 103 | - | |
| 104 | - User-agent: ChatGLM-Spider | |
| 105 | - Disallow: / | |
| 106 | - | |
| 107 | - User-agent: ChatGPT-User | |
| 108 | - Disallow: / | |
| 109 | - | |
| 110 | - User-agent: Claude-SearchBot | |
| 111 | - Disallow: / | |
| 112 | - | |
| 113 | - User-agent: Claude-User | |
| 114 | - Disallow: / | |
| 115 | - | |
| 116 | - User-agent: Claude-Web | |
| 117 | - Disallow: / | |
| 118 | - | |
| 119 | - User-agent: ClaudeBot | |
| 120 | - Disallow: / | |
| 121 | - | |
| 122 | - User-agent: cohere-ai | |
| 123 | - Disallow: / | |
| 124 | - | |
| 125 | - User-agent: Cotoyogi | |
| 126 | - Disallow: / | |
| 127 | - | |
| 128 | - User-agent: DeepSeekBot | |
| 129 | - Disallow: / | |
| 130 | - | |
| 131 | - User-agent: Diffbot | |
| 132 | - Disallow: / | |
| 133 | - | |
| 134 | - User-agent: DuckAssistBot | |
| 135 | - Disallow: / | |
| 136 | - | |
| 137 | - User-agent: FacebookBot | |
| 138 | - Disallow: / | |
| 139 | - | |
| 140 | - User-agent: FriendlyCrawler | |
| 141 | - Disallow: / | |
| 142 | - | |
| 143 | - User-agent: Google-CloudVertexBot | |
| 144 | - Disallow: / | |
| 145 | - | |
| 146 | - User-agent: Google-Extended | |
| 147 | - Allow: / | |
| 148 | - | |
| 149 | - User-agent: GoogleOther | |
| 150 | - Disallow: / | |
| 151 | - | |
| 152 | - User-agent: GoogleOther-Image | |
| 153 | - Disallow: / | |
| 154 | - | |
| 155 | - User-agent: GoogleOther-Video | |
| 156 | - Disallow: / | |
| 157 | - | |
| 158 | - User-agent: GPTBot | |
| 159 | - Disallow: / | |
| 160 | - | |
| 161 | - User-agent: Grok | |
| 162 | - Disallow: / | |
| 163 | - | |
| 164 | - User-agent: iaskspider/2.0 | |
| 165 | - Disallow: / | |
| 166 | - | |
| 167 | - User-agent: ICC-Crawler | |
| 168 | - Disallow: / | |
| 169 | - | |
| 170 | - User-agent: ImagesiftBot | |
| 171 | - Disallow: / | |
| 172 | - | |
| 173 | - User-agent: img2dataset | |
| 174 | - Disallow: / | |
| 175 | - | |
| 176 | - User-agent: ISSCyberRiskCrawler | |
| 177 | - Disallow: / | |
| 178 | - | |
| 179 | - User-agent: Kangaroo Bot | |
| 180 | - Disallow: / | |
| 181 | - | |
| 182 | - User-agent: KunatoCrawler | |
| 183 | - Disallow: / | |
| 184 | - | |
| 185 | - User-agent: Meltwater | |
| 186 | - Disallow: / | |
| 187 | - | |
| 188 | - User-agent: Meta-ExternalAgent | |
| 189 | - Disallow: / | |
| 190 | - | |
| 191 | - User-agent: Meta-ExternalFetcher | |
| 192 | - Disallow: / | |
| 193 | - | |
| 194 | - User-agent: MistralAI-User | |
| 195 | - Disallow: / | |
| 196 | - | |
| 197 | - User-agent: OAI-Operator | |
| 198 | - Disallow: / | |
| 199 | - | |
| 200 | - User-agent: OAI-SearchBot | |
| 201 | - Disallow: / | |
| 202 | - | |
| 203 | - User-agent: omgili | |
| 204 | - Disallow: / | |
| 205 | - | |
| 206 | - User-agent: omgilibot | |
| 207 | - Disallow: / | |
| 208 | - | |
| 209 | - User-agent: PanguBot | |
| 210 | - Disallow: / | |
| 211 | - | |
| 212 | - User-agent: Perplexity-User | |
| 213 | - Disallow: / | |
| 214 | - | |
| 215 | - User-agent: PerplexityBot | |
| 216 | - Disallow: / | |
| 217 | - | |
| 218 | - User-agent: PetalBot | |
| 219 | - Disallow: / | |
| 220 | - | |
| 221 | - User-agent: QualifiedBot | |
| 222 | - Disallow: / | |
| 223 | - | |
| 224 | - User-agent: Scrapy | |
| 225 | - Disallow: / | |
| 226 | - | |
| 227 | - User-agent: Seekr | |
| 228 | - Disallow: / | |
| 229 | - | |
| 230 | - User-agent: Sidetrade indexer bot | |
| 231 | - Disallow: / | |
| 232 | - | |
| 233 | - User-agent: TikTokSpider | |
| 234 | - Disallow: / | |
| 235 | - | |
| 236 | - User-agent: Timpibot | |
| 237 | - Disallow: / | |
| 238 | - | |
| 239 | - User-agent: VelenPublicWebCrawler | |
| 240 | - Disallow: / | |
| 241 | - | |
| 242 | - User-agent: WARDBot | |
| 243 | - Disallow: / | |
| 244 | - | |
| 245 | - User-agent: Webzio-Extended | |
| 246 | - Disallow: / | |
| 247 | - | |
| 248 | - User-agent: WRTNBot | |
| 249 | - Disallow: / | |
| 250 | - | |
| 251 | - | |
| 252 | - #Sitemap URLs | |
| 116 | + # Sitemap URLs | |
| 253 | 117 | Sitemap: https://www.telegraph.co.uk/sitemap.xml |
| 254 | 118 | Sitemap: https://www.telegraph.co.uk/custom/authors/sitemap.xml |
| 255 | 119 | Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml |
| 256 | 120 | Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml |
| 257 | - Sitemap: https://www.telegraph.co.uk/vouchercodes/sitemap.xml | |
| 258 | 121 | Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml |
| 259 | 122 | |
| 123 | + | |
| 124 | + | |
| 125 | + |