Delta Air Lines / robots.txt
← back to delta.com · change detected 8d ago (2026-10-29T17:28:06Z) · modified +67 −12
What changed
- Added rules for user-agent ‘ai2-crawler’
- 🤖 AIAdded rules for AI crawler Amazonbot (Amazon)
- 🤖 AIAdded rules for AI crawler anthropic-ai (Anthropic)
- 🤖 AIAdded rules for AI crawler Applebot-Extended (Apple)
- 🤖 AIAdded rules for AI crawler Bytespider (ByteDance)
- 🤖 AIAdded rules for AI crawler CCBot (Common Crawl)
- 🤖 AIAdded rules for AI crawler ChatGPT-User (OpenAI)
- 🤖 AIAdded rules for AI crawler ClaudeBot (Anthropic)
- 🤖 AIAdded rules for AI crawler cohere-ai (Cohere)
- 🤖 AIAdded rules for AI crawler Diffbot (Diffbot)
- 🤖 AIAdded rules for AI crawler DuckAssistBot (DuckDuckGo)
- 🤖 AIAdded rules for AI crawler FacebookBot (Meta)
- 🤖 AIAdded rules for AI crawler Google-Extended (Google)
- 🤖 AIAdded rules for AI crawler GPTBot (OpenAI)
- 🤖 AIAdded rules for AI crawler meta-externalagent (Meta)
- 🤖 AIAdded rules for AI crawler meta-externalfetcher (Meta)
- 🤖 AIAdded rules for AI crawler OAI-SearchBot (OpenAI)
- 🤖 AIAdded rules for AI crawler PerplexityBot (Perplexity)
- 🤖 AIAdded rules for AI crawler YouBot (You.com)
- Added Allow: /en_US/skymiles/medallion-program/reclaim-my-status.html for *
- Added Allow: /en_US/skymiles/medallion-program/reclaim-my-status/enrollment.html for *
- Added Disallow: /_jcr_content/ for *
- Added Disallow: /edoc/displayEdoc* for *
- Added Disallow: /en_US/ for *
- Added Disallow: /flight-search/search? for *
- Added Disallow: /jcr:content/ for *
- Removed Allow: /profile/enrolllanding.action for *
- Removed Disallow: /*?*cacheKeySuffix= for *
- Removed Disallow: /*cacheKey* for *
- Removed Disallow: /edoc/displayEdocTNC.action* for *
- Removed Disallow: /edoc/displayEdocTnC.action* for *
- Removed Disallow: /flight-search/search?&tripType= for *
- Removed Disallow: /flight-search/search?action= for *
- Removed Disallow: /flight-status for *
- Removed Disallow: /flight-status/ for *
- Removed Disallow: /flight-status/* for *
- Removed Disallow: /home for *
Diff old (2026-07-14T17:01:50Z) → new (2026-10-29T17:28:06Z)
| @@ -1,10 +1,71 @@ | ||
| 1 | - User-agent: * | |
| 1 | + # - LLM / AI Crawlers - Explicitly Permitted - | |
| 2 | + User-agent: GPTBot | |
| 3 | + Allow: / | |
| 4 | + | |
| 5 | + User-agent: ChatGPT-User | |
| 6 | + Allow: / | |
| 7 | + | |
| 8 | + User-agent: OAI-SearchBot | |
| 9 | + Allow: / | |
| 10 | + | |
| 11 | + User-agent: ClaudeBot | |
| 12 | + Allow: / | |
| 13 | + | |
| 14 | + User-agent: anthropic-ai | |
| 15 | + Allow: / | |
| 16 | + | |
| 17 | + User-agent: Google-Extended | |
| 18 | + Allow: / | |
| 19 | + | |
| 20 | + User-agent: FacebookBot | |
| 21 | + Allow: / | |
| 22 | + | |
| 23 | + User-agent: meta-externalagent | |
| 24 | + Allow: / | |
| 25 | + | |
| 26 | + User-agent: Meta-ExternalFetcher | |
| 27 | + Allow: / | |
| 28 | + | |
| 29 | + User-agent: Applebot-Extended | |
| 30 | + Allow: / | |
| 31 | + | |
| 32 | + User-agent: Amazonbot | |
| 33 | + Allow: / | |
| 34 | + | |
| 35 | + User-agent: PerplexityBot | |
| 36 | + Allow: / | |
| 37 | + | |
| 38 | + User-agent: CCBot | |
| 39 | + Allow: / | |
| 40 | + | |
| 41 | + User-agent: cohere-ai | |
| 42 | + Allow: / | |
| 43 | + | |
| 44 | + User-agent: Bytespider | |
| 45 | + Allow: / | |
| 46 | + | |
| 47 | + User-agent: DuckAssistBot | |
| 48 | + Allow: / | |
| 49 | + | |
| 50 | + User-agent: YouBot | |
| 51 | + Allow: / | |
| 52 | + | |
| 53 | + User-agent: Diffbot | |
| 54 | + Allow: / | |
| 55 | + | |
| 56 | + User-agent: ai2-crawler | |
| 57 | + Allow: / | |
| 2 | 58 | |
| 3 | 59 | # If you are reading this looking for a means of reporting a security issue, please see security.txt @ https://www.delta.com/us/en/legal/vulnerability-disclosure-guidelines |
| 4 | 60 | |
| 61 | + User-agent: * | |
| 62 | + | |
| 63 | + Allow: /en_US/skymiles/medallion-program/reclaim-my-status.html | |
| 64 | + Allow: /en_US/skymiles/medallion-program/reclaim-my-status/enrollment.html | |
| 65 | + Disallow: /en_US/ | |
| 66 | + | |
| 5 | 67 | # Allow Individual Pages |
| 6 | 68 | Allow: /egift/eGiftPurchase.action |
| 7 | - Allow: /profile/enrolllanding.action | |
| 8 | 69 | Allow: /etc/designs/global/clientlib-overview.css |
| 9 | 70 | |
| 10 | 71 | # Allow Individual .jsp Pages |
| @@ -52,25 +113,19 @@ | ||
| 52 | 113 | Disallow: /*.pdf |
| 53 | 114 | |
| 54 | 115 | # Disallow generated pages |
| 55 | - Disallow: /flight-status/ | |
| 56 | - Disallow: /flight-status | |
| 57 | 116 | Disallow: *flight-status* |
| 58 | - Disallow: /flight-status/* | |
| 59 | 117 | Disallow: *cacheKey* |
| 60 | - Disallow: /*cacheKey* | |
| 61 | - Disallow: /*?*cacheKeySuffix= | |
| 62 | 118 | Disallow: *?*paymentRef= |
| 63 | 119 | Disallow: /site-search?searchTerm=* |
| 64 | 120 | Disallow: /profile/enrolllanding.action?partnerCd* |
| 65 | 121 | Disallow: /merch/searchHotelAction.action?locationCode* |
| 66 | 122 | Disallow: /merch/searchCarAction.action?locationCode* |
| 67 | - Disallow: /edoc/displayEdocTNC.action* | |
| 123 | + Disallow: /edoc/displayEdoc* | |
| 68 | 124 | Disallow: /edoc/redeemDocuments.action?action=* |
| 69 | 125 | Disallow: /air-shopping/findFlights.action* |
| 70 | 126 | Disallow: /findFlights.action* |
| 71 | 127 | Disallow: /flight-search/book-a-flight?cacheKeySuffix= |
| 72 | - Disallow: /flight-search/search?&tripType= | |
| 73 | - Disallow: /flight-search/search?action= | |
| 128 | + Disallow: /flight-search/search? | |
| 74 | 129 | Disallow: /custlogin/showResetPassword.action* |
| 75 | 130 | Disallow: /flight-search-2/ |
| 76 | 131 | Disallow: /search* |
| @@ -86,14 +141,12 @@ | ||
| 86 | 141 | Disallow: /reshop/ |
| 87 | 142 | Disallow: /cartreshop/activity/reshopPurchase.action |
| 88 | 143 | Disallow: /contactus/pages/business_programs_services/agency_irop/index.jsp |
| 89 | - Disallow: /edoc/displayEdocTnC.action* | |
| 90 | 144 | Disallow: /mypromos/registerPromotion.action |
| 91 | 145 | Disallow: /profile/promotionconfirm.action* |
| 92 | 146 | Disallow: /skyclub/purchasemembership_removePromotion.action |
| 93 | 147 | Disallow: /profile/enrollconfirm.action |
| 94 | 148 | |
| 95 | 149 | # Disallow Subfolders |
| 96 | - Disallow: /home | |
| 97 | 150 | Disallow: /home/ |
| 98 | 151 | Disallow: /completepurchase |
| 99 | 152 | Disallow: /complete-purchase |
| @@ -125,6 +178,8 @@ | ||
| 125 | 178 | Disallow: /air-shopping/* |
| 126 | 179 | Disallow: /air-shopping* |
| 127 | 180 | Disallow: /agency-refund/ |
| 181 | + Disallow: /_jcr_content/ | |
| 182 | + Disallow: /jcr:content/ | |
| 128 | 183 | |
| 129 | 184 | # Disallow Individual App Pages |
| 130 | 185 | Disallow: /book/activity/confirmation.action |