The Telegraph / robots.txt
← back to telegraph.co.uk · change detected 3d ago (2026-11-03T13:00:12Z) · modified +3 −70
What changed
- Added Allow: / for *
- Added Disallow: /promotions/emails/ for *
- Added Disallow: /sponsored/travel/msc-cruises/ for *
- 🤖 AIRemoved Allow: /bah-avis/usa-sports-road-trip-map for AI2Bot
- 🤖 AIRemoved Allow: /bespoke-content/ for AI2Bot
- 🤖 AIRemoved Allow: /betting/ for AI2Bot
- 🤖 AIRemoved Allow: /brand-usa for AI2Bot
- 🤖 AIRemoved Allow: /branded-content/ for AI2Bot
- 🤖 AIRemoved Allow: /british-airways-holidays/cyprus-hotel-map for AI2Bot
- 🤖 AIRemoved Allow: /business/ai-era-job-skills/ for AI2Bot
- 🤖 AIRemoved Allow: /business/ai-workflow-transformation/ for AI2Bot
- 🤖 AIRemoved Allow: /business/future-hr-technology/ for AI2Bot
- 🤖 AIRemoved Allow: /business/golf-dp-world-tour/ for AI2Bot
- 🤖 AIRemoved Allow: /business/search-marketing/ for AI2Bot
- 🤖 AIRemoved Allow: /business/waste-management/ for AI2Bot
- 🤖 AIRemoved Allow: /deals/ for AI2Bot
- 🤖 AIRemoved Allow: /environment/woodland-action/ for AI2Bot
- 🤖 AIRemoved Allow: /fashion/joseph-ribkoff-autumn-collection/ for AI2Bot
- 🤖 AIRemoved Allow: /financial-services/ for AI2Bot
- 🤖 AIRemoved Allow: /food-and-drink/m-and-s-seasonal-favourites/ for AI2Bot
- 🤖 AIRemoved Allow: /food-and-drink/m-and-s-seasonal-favourites/organic-range/ for AI2Bot
- 🤖 AIRemoved Allow: /food-and-drink/maldon-salt-perfect-pyramid-challenge/ for AI2Bot
- 🤖 AIRemoved Allow: /food-and-drink/spanish-dining-uk/ for AI2Bot
- 🤖 AIRemoved Allow: /football/global-football-industry-awards/ for AI2Bot
- 🤖 AIRemoved Allow: /health-fitness/how-to-choose-private-hospital-hca-healthcare/ for AI2Bot
- 🤖 AIRemoved Allow: /interiors/next-home-autumn-interiors-trends/ for AI2Bot
- 🤖 AIRemoved Allow: /junior-golf/ for AI2Bot
- 🤖 AIRemoved Allow: /lifestyle-solutions/ for AI2Bot
- 🤖 AIRemoved Allow: /lifestyle/apl-media/ for AI2Bot
- 🤖 AIRemoved Allow: /money/investing/fc-investment-trust/long-term-investing/ for AI2Bot
- 🤖 AIRemoved Allow: /rugby-union/grassroots-rugby-heroes/ for AI2Bot
- 🤖 AIRemoved Allow: /rugby-union/levelling-the-playing-field/ for AI2Bot
- 🤖 AIRemoved Allow: /sailgp/inside-f50-catamaran for AI2Bot
- 🤖 AIRemoved Allow: /seabourn/explore-the-mediterranean for AI2Bot
- 🤖 AIRemoved Allow: /sport/sailgp-championship/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/active-family-holidays/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/away-holidays/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/barbados-holiday-guide/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/british-airways-american-express/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/canadian-affair-holidays/rocky-mountaineer-train-route/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/croatia-holidays/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/cruises/holland-america-line-luxury-holidays/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/cruises/seabourn-luxury-cruises/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/discover-america/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/exclusiveoffers/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/florida-guide/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/fly-drive-holidays/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/grenada-guide/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/icons-of-the-sky/ for AI2Bot
- 🤖 AIRemoved Allow: /travel/indian-ocean-escapes/ for AI2Bot
- ... and 3652 more change(s)
Diff old (2026-10-22T21:05:52Z) → new (2026-11-03T13:00:12Z)
| @@ -29,7 +29,9 @@ | ||
| 29 | 29 | Disallow: /application/* |
| 30 | 30 | Disallow: /ixale/ |
| 31 | 31 | Disallow: /core/Content/ |
| 32 | + Disallow: /promotions/emails/ | |
| 32 | 33 | Disallow: /r/ |
| 34 | + Disallow: /sponsored/travel/msc-cruises/ | |
| 33 | 35 | Disallow: /travel/hotel/e/* |
| 34 | 36 | Disallow: /travel/hotels/hotel-finder/ |
| 35 | 37 | Disallow: /podcasts-more/ |
| @@ -42,6 +44,7 @@ | ||
| 42 | 44 | Disallow: /*&p= |
| 43 | 45 | Disallow: /customer/subscription/*? |
| 44 | 46 | Disallow: /edge-redirection/ |
| 47 | + Allow: / | |
| 45 | 48 | |
| 46 | 49 | |
| 47 | 50 | # Bots which make unnecessary bot traffic |
| @@ -110,75 +113,6 @@ | ||
| 110 | 113 | Allow: /travel/luxury-stays/ |
| 111 | 114 | Allow: /business/uk-business-growth/ |
| 112 | 115 | Allow: /ads.txt |
| 113 | - Allow: /bah-avis/usa-sports-road-trip-map | |
| 114 | - Allow: /betting/ | |
| 115 | - Allow: /british-airways-holidays/cyprus-hotel-map | |
| 116 | - Allow: /business/ai-era-job-skills/ | |
| 117 | - Allow: /business/ai-workflow-transformation/ | |
| 118 | - Allow: /business/future-hr-technology/ | |
| 119 | - Allow: /business/golf-dp-world-tour/ | |
| 120 | - Allow: /business/search-marketing/ | |
| 121 | - Allow: /business/waste-management/ | |
| 122 | - Allow: /deals/ | |
| 123 | - Allow: /environment/woodland-action/ | |
| 124 | - Allow: /fashion/joseph-ribkoff-autumn-collection/ | |
| 125 | - Allow: /financial-services/ | |
| 126 | - Allow: /food-and-drink/m-and-s-seasonal-favourites/ | |
| 127 | - Allow: /food-and-drink/m-and-s-seasonal-favourites/organic-range/ | |
| 128 | - Allow: /food-and-drink/maldon-salt-perfect-pyramid-challenge/ | |
| 129 | - Allow: /food-and-drink/spanish-dining-uk/ | |
| 130 | - Allow: /football/global-football-industry-awards/ | |
| 131 | - Allow: /health-fitness/how-to-choose-private-hospital-hca-healthcare/ | |
| 132 | - Allow: /interiors/next-home-autumn-interiors-trends/ | |
| 133 | - Allow: /junior-golf/ | |
| 134 | - Allow: /lifestyle-solutions/ | |
| 135 | - Allow: /lifestyle/apl-media/ | |
| 136 | - Allow: /branded-content/ | |
| 137 | - Allow: /money/investing/fc-investment-trust/long-term-investing/ | |
| 138 | - Allow: /branded-content/ | |
| 139 | - Allow: /rugby-union/grassroots-rugby-heroes/ | |
| 140 | - Allow: /rugby-union/levelling-the-playing-field/ | |
| 141 | - Allow: /sailgp/inside-f50-catamaran | |
| 142 | - Allow: /seabourn/explore-the-mediterranean | |
| 143 | - Allow: /sport/sailgp-championship/ | |
| 144 | - Allow: /travel/active-family-holidays/ | |
| 145 | - Allow: /travel/away-holidays/ | |
| 146 | - Allow: /travel/barbados-holiday-guide/ | |
| 147 | - Allow: /travel/british-airways-american-express/ | |
| 148 | - Allow: /travel/canadian-affair-holidays/rocky-mountaineer-train-route/ | |
| 149 | - Allow: /travel/croatia-holidays/ | |
| 150 | - Allow: /travel/cruises/holland-america-line-luxury-holidays/ | |
| 151 | - Allow: /travel/cruises/seabourn-luxury-cruises/ | |
| 152 | - Allow: /travel/discover-america/ | |
| 153 | - Allow: /travel/exclusiveoffers/ | |
| 154 | - Allow: /branded-content/ | |
| 155 | - Allow: /travel/florida-guide/ | |
| 156 | - Allow: /travel/fly-drive-holidays/ | |
| 157 | - Allow: /travel/grenada-guide/ | |
| 158 | - Allow: /travel/icons-of-the-sky/ | |
| 159 | - Allow: /travel/indian-ocean-escapes/ | |
| 160 | - Allow: /travel/indulge-in-ireland/ | |
| 161 | - Allow: /travel/jamaica-holiday-guide/ | |
| 162 | - Allow: /travel/japan-ninja-heritage/ | |
| 163 | - Allow: /travel/luxury-all-inclusive-resorts/affordable-caribbean-hotels/ | |
| 164 | - Allow: /travel/luxury-holidays/ | |
| 165 | - Allow: /travel/luxury-hotel-getaways/ | |
| 166 | - Allow: /travel/luxury-island-escapes/ | |
| 167 | - Allow: /travel/maldives-holiday-guide/ | |
| 168 | - Allow: /travel/ski-resorts-for-everyone/ | |
| 169 | - Allow: /travel/spark/best-of-switzerland/ | |
| 170 | - Allow: /travel/tailor-made-tours/ | |
| 171 | - Allow: /travel/tenerife-holidays/ | |
| 172 | - Allow: /travel/tips-for-visiting-spain/ | |
| 173 | - Allow: /travel/travel-counsellors/book-a-cruise-with-travel-counsellors/ | |
| 174 | - Allow: /travel/travel-solutions/ | |
| 175 | - Allow: /travel/visit-cayman-islands/ | |
| 176 | - Allow: /travel/why-choose-malta/ | |
| 177 | - Allow: /trusted-partners/ | |
| 178 | - Allow: /brand-usa | |
| 179 | - Allow: /travel/visit-aarhus/ | |
| 180 | - Allow: /bespoke-content/ | |
| 181 | - | |
| 182 | 116 | |
| 183 | 117 | |
| 184 | 118 | # Sitemap URLs |
| @@ -187,4 +121,3 @@ | ||
| 187 | 121 | Sitemap: https://www.telegraph.co.uk/custom/main-pages/sitemap.xml |
| 188 | 122 | Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml |
| 189 | 123 | Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml |
| 190 | - |