NeuralCrawl

Expedia Group / robots.txt snapshot

← back to expedia.com · fetched 2026-09-28T18:34:49Z (1mo ago) · HTTP 200 · 2210 bytes · sha256 3d5c2166f871be02 · raw

final URL: https://www.expedia.com/robots.txt

1#
2# General bots
3#
4user-agent: *
5#hotel
6disallow: /Hotel-Search
7disallow: /hotel-search
8disallow: /search?
9disallow: /*/search?
10disallow: /ugc/urs/api/hotelreviews/hotel/HOTELID/
11disallow: /LastBooking/ElapsedTime
12disallow: /*chkin=*
13disallow: /*chkout=*
14disallow: /HotelCheckout
15disallow: /Checkout
16disallow: /checkout
17disallow: /HotelBookingConfirmation
18disallow: /en-au/
19disallow: /en-nz/
20#flight
21disallow: /Flights-Search
22disallow: /Flight-Information
23disallow: /Flight-SearchResults
24disallow: /Flights-BagFees
25disallow: /Details
26disallow: /FlightCheckout
27disallow: /Confirmation-Flight
28#package
29disallow: /MultiItemCheckout
30disallow: /MultiItemBookingConfirmation
31disallow: /flexibleshopping
32#car
33disallow: /carsearch
34disallow: /CarCheckout
35#activities
36disallow: /things-to-do/search?
37disallow: /things-to-do/*/search?
38disallow: /*/things-to-do/search
39disallow: /things-to-do/reviews-external$
40#cruise
41disallow: /Cruise-Search
42disallow: /Cruise-CabinCategory
43disallow: /CruiseCheckout
44#other
45disallow: /legacyGOTO
46disallow: /go/
47disallow: /info/
48disallow: /trips/
49disallow: /Search-Results
50disallow: /user/createaccount
51disallow: /signup?*
52disallow: /login?*
53#test-seo
54disallow: /api/uisprime/track
55disallow: /trvl-px/
56disallow: /egcs/v2/collect
57disallow: /api/ucs/shortlist/
58disallow: /api/v4/typeahead/
59disallow: /p/home
60disallow: /*/p/home
61disallow: /auth/
62disallow: /*/auth/
63disallow: /*/sso/
64# /cl/* endpoints removal test
65Disallow: /cl/*
66
67#
68# Ads
69#
70user-agent: google-hoteladsverifier
71user-agent: Mediapartners-Google
72user-agent: AdsBot-Google-Mobile
73user-agent: AdsBot-Google
74user-agent: AdIdxBot
75user-agent: OAI-AdsBot
76allow: /
77#test-seo
78disallow: /api/uisprime/track
79disallow: /trvl-px/
80disallow: /egcs/v2/collect
81disallow: /api/ucs/shortlist/
82disallow: /api/v4/typeahead/
83# /cl/* endpoints removal test
84Disallow: /cl/*
85
86#
87# SemrushBot
88#
89user-agent: SemrushBot
90disallow: /
91
92#
93# AI Search
94#
95user-agent: OAI-SearchBot
96user-agent: ChatGPT-User
97user-agent: PerplexityBot
98user-agent: Perplexity-User
99user-agent: Claude-User
100user-agent: Claude-SearchBot
101allow: /
102disallow: /p/home
103disallow: /*/p/home
104disallow: /auth/
105disallow: /*/auth/
106disallow: /*/sso/
107# /cl/* endpoints removal test
108Disallow: /cl/*