The Telegraph / robots.txt
← back to telegraph.co.uk · change detected 15d ago (2026-10-22T21:05:52Z) · modified +69 −7
What changed
- Removed Disallow: /promotions/emails/ for *
- Removed Disallow: /sponsored/travel/msc-cruises/ for *
- 🤖 AIAdded Allow: /bah-avis/usa-sports-road-trip-map for AI2Bot
- 🤖 AIAdded Allow: /bespoke-content/ for AI2Bot
- 🤖 AIAdded Allow: /betting/ for AI2Bot
- 🤖 AIAdded Allow: /brand-usa for AI2Bot
- 🤖 AIAdded Allow: /branded-content/ for AI2Bot
- 🤖 AIAdded Allow: /british-airways-holidays/cyprus-hotel-map for AI2Bot
- 🤖 AIAdded Allow: /business/ai-era-job-skills/ for AI2Bot
- 🤖 AIAdded Allow: /business/ai-workflow-transformation/ for AI2Bot
- 🤖 AIAdded Allow: /business/future-hr-technology/ for AI2Bot
- 🤖 AIAdded Allow: /business/golf-dp-world-tour/ for AI2Bot
- 🤖 AIAdded Allow: /business/search-marketing/ for AI2Bot
- 🤖 AIAdded Allow: /business/waste-management/ for AI2Bot
- 🤖 AIAdded Allow: /deals/ for AI2Bot
- 🤖 AIAdded Allow: /environment/woodland-action/ for AI2Bot
- 🤖 AIAdded Allow: /fashion/joseph-ribkoff-autumn-collection/ for AI2Bot
- 🤖 AIAdded Allow: /financial-services/ for AI2Bot
- 🤖 AIAdded Allow: /food-and-drink/m-and-s-seasonal-favourites/ for AI2Bot
- 🤖 AIAdded Allow: /food-and-drink/m-and-s-seasonal-favourites/organic-range/ for AI2Bot
- 🤖 AIAdded Allow: /food-and-drink/maldon-salt-perfect-pyramid-challenge/ for AI2Bot
- 🤖 AIAdded Allow: /food-and-drink/spanish-dining-uk/ for AI2Bot
- 🤖 AIAdded Allow: /football/global-football-industry-awards/ for AI2Bot
- 🤖 AIAdded Allow: /health-fitness/how-to-choose-private-hospital-hca-healthcare/ for AI2Bot
- 🤖 AIAdded Allow: /interiors/next-home-autumn-interiors-trends/ for AI2Bot
- 🤖 AIAdded Allow: /junior-golf/ for AI2Bot
- 🤖 AIAdded Allow: /lifestyle-solutions/ for AI2Bot
- 🤖 AIAdded Allow: /lifestyle/apl-media/ for AI2Bot
- 🤖 AIAdded Allow: /money/investing/fc-investment-trust/long-term-investing/ for AI2Bot
- 🤖 AIAdded Allow: /rugby-union/grassroots-rugby-heroes/ for AI2Bot
- 🤖 AIAdded Allow: /rugby-union/levelling-the-playing-field/ for AI2Bot
- 🤖 AIAdded Allow: /sailgp/inside-f50-catamaran for AI2Bot
- 🤖 AIAdded Allow: /seabourn/explore-the-mediterranean for AI2Bot
- 🤖 AIAdded Allow: /sport/sailgp-championship/ for AI2Bot
- 🤖 AIAdded Allow: /travel/active-family-holidays/ for AI2Bot
- 🤖 AIAdded Allow: /travel/away-holidays/ for AI2Bot
- 🤖 AIAdded Allow: /travel/barbados-holiday-guide/ for AI2Bot
- 🤖 AIAdded Allow: /travel/british-airways-american-express/ for AI2Bot
- 🤖 AIAdded Allow: /travel/canadian-affair-holidays/rocky-mountaineer-train-route/ for AI2Bot
- 🤖 AIAdded Allow: /travel/croatia-holidays/ for AI2Bot
- 🤖 AIAdded Allow: /travel/cruises/holland-america-line-luxury-holidays/ for AI2Bot
- 🤖 AIAdded Allow: /travel/cruises/seabourn-luxury-cruises/ for AI2Bot
- 🤖 AIAdded Allow: /travel/discover-america/ for AI2Bot
- 🤖 AIAdded Allow: /travel/exclusiveoffers/ for AI2Bot
- 🤖 AIAdded Allow: /travel/florida-guide/ for AI2Bot
- 🤖 AIAdded Allow: /travel/fly-drive-holidays/ for AI2Bot
- 🤖 AIAdded Allow: /travel/grenada-guide/ for AI2Bot
- 🤖 AIAdded Allow: /travel/icons-of-the-sky/ for AI2Bot
- 🤖 AIAdded Allow: /travel/indian-ocean-escapes/ for AI2Bot
- 🤖 AIAdded Allow: /travel/indulge-in-ireland/ for AI2Bot
- ... and 3650 more change(s)
Diff old (2026-10-08T10:00:34Z) → new (2026-10-22T21:05:52Z)
| @@ -29,9 +29,7 @@ | ||
| 29 | 29 | Disallow: /application/* |
| 30 | 30 | Disallow: /ixale/ |
| 31 | 31 | Disallow: /core/Content/ |
| 32 | - Disallow: /promotions/emails/ | |
| 33 | 32 | Disallow: /r/ |
| 34 | - Disallow: /sponsored/travel/msc-cruises/ | |
| 35 | 33 | Disallow: /travel/hotel/e/* |
| 36 | 34 | Disallow: /travel/hotels/hotel-finder/ |
| 37 | 35 | Disallow: /podcasts-more/ |
| @@ -112,6 +110,75 @@ | ||
| 112 | 110 | Allow: /travel/luxury-stays/ |
| 113 | 111 | Allow: /business/uk-business-growth/ |
| 114 | 112 | Allow: /ads.txt |
| 113 | + Allow: /bah-avis/usa-sports-road-trip-map | |
| 114 | + Allow: /betting/ | |
| 115 | + Allow: /british-airways-holidays/cyprus-hotel-map | |
| 116 | + Allow: /business/ai-era-job-skills/ | |
| 117 | + Allow: /business/ai-workflow-transformation/ | |
| 118 | + Allow: /business/future-hr-technology/ | |
| 119 | + Allow: /business/golf-dp-world-tour/ | |
| 120 | + Allow: /business/search-marketing/ | |
| 121 | + Allow: /business/waste-management/ | |
| 122 | + Allow: /deals/ | |
| 123 | + Allow: /environment/woodland-action/ | |
| 124 | + Allow: /fashion/joseph-ribkoff-autumn-collection/ | |
| 125 | + Allow: /financial-services/ | |
| 126 | + Allow: /food-and-drink/m-and-s-seasonal-favourites/ | |
| 127 | + Allow: /food-and-drink/m-and-s-seasonal-favourites/organic-range/ | |
| 128 | + Allow: /food-and-drink/maldon-salt-perfect-pyramid-challenge/ | |
| 129 | + Allow: /food-and-drink/spanish-dining-uk/ | |
| 130 | + Allow: /football/global-football-industry-awards/ | |
| 131 | + Allow: /health-fitness/how-to-choose-private-hospital-hca-healthcare/ | |
| 132 | + Allow: /interiors/next-home-autumn-interiors-trends/ | |
| 133 | + Allow: /junior-golf/ | |
| 134 | + Allow: /lifestyle-solutions/ | |
| 135 | + Allow: /lifestyle/apl-media/ | |
| 136 | + Allow: /branded-content/ | |
| 137 | + Allow: /money/investing/fc-investment-trust/long-term-investing/ | |
| 138 | + Allow: /branded-content/ | |
| 139 | + Allow: /rugby-union/grassroots-rugby-heroes/ | |
| 140 | + Allow: /rugby-union/levelling-the-playing-field/ | |
| 141 | + Allow: /sailgp/inside-f50-catamaran | |
| 142 | + Allow: /seabourn/explore-the-mediterranean | |
| 143 | + Allow: /sport/sailgp-championship/ | |
| 144 | + Allow: /travel/active-family-holidays/ | |
| 145 | + Allow: /travel/away-holidays/ | |
| 146 | + Allow: /travel/barbados-holiday-guide/ | |
| 147 | + Allow: /travel/british-airways-american-express/ | |
| 148 | + Allow: /travel/canadian-affair-holidays/rocky-mountaineer-train-route/ | |
| 149 | + Allow: /travel/croatia-holidays/ | |
| 150 | + Allow: /travel/cruises/holland-america-line-luxury-holidays/ | |
| 151 | + Allow: /travel/cruises/seabourn-luxury-cruises/ | |
| 152 | + Allow: /travel/discover-america/ | |
| 153 | + Allow: /travel/exclusiveoffers/ | |
| 154 | + Allow: /branded-content/ | |
| 155 | + Allow: /travel/florida-guide/ | |
| 156 | + Allow: /travel/fly-drive-holidays/ | |
| 157 | + Allow: /travel/grenada-guide/ | |
| 158 | + Allow: /travel/icons-of-the-sky/ | |
| 159 | + Allow: /travel/indian-ocean-escapes/ | |
| 160 | + Allow: /travel/indulge-in-ireland/ | |
| 161 | + Allow: /travel/jamaica-holiday-guide/ | |
| 162 | + Allow: /travel/japan-ninja-heritage/ | |
| 163 | + Allow: /travel/luxury-all-inclusive-resorts/affordable-caribbean-hotels/ | |
| 164 | + Allow: /travel/luxury-holidays/ | |
| 165 | + Allow: /travel/luxury-hotel-getaways/ | |
| 166 | + Allow: /travel/luxury-island-escapes/ | |
| 167 | + Allow: /travel/maldives-holiday-guide/ | |
| 168 | + Allow: /travel/ski-resorts-for-everyone/ | |
| 169 | + Allow: /travel/spark/best-of-switzerland/ | |
| 170 | + Allow: /travel/tailor-made-tours/ | |
| 171 | + Allow: /travel/tenerife-holidays/ | |
| 172 | + Allow: /travel/tips-for-visiting-spain/ | |
| 173 | + Allow: /travel/travel-counsellors/book-a-cruise-with-travel-counsellors/ | |
| 174 | + Allow: /travel/travel-solutions/ | |
| 175 | + Allow: /travel/visit-cayman-islands/ | |
| 176 | + Allow: /travel/why-choose-malta/ | |
| 177 | + Allow: /trusted-partners/ | |
| 178 | + Allow: /brand-usa | |
| 179 | + Allow: /travel/visit-aarhus/ | |
| 180 | + Allow: /bespoke-content/ | |
| 181 | + | |
| 115 | 182 | |
| 116 | 183 | |
| 117 | 184 | # Sitemap URLs |
| @@ -121,8 +188,3 @@ | ||
| 121 | 188 | Sitemap: https://www.telegraph.co.uk/custom/daily-news/sitemap.xml |
| 122 | 189 | Sitemap: https://www.telegraph.co.uk/custom/author-pages/sitemap.xml |
| 123 | 190 | |
| 124 | - | |
| 125 | - | |
| 126 | - | |
| 127 | - | |
| 128 | - |