NeuralCrawl

Google / robots.txt snapshot

← back to google.com · fetched 2026-06-20T14:56:27Z (7h ago) · HTTP 200 · 6607 bytes · sha256 e93753d01f34ae18 · raw

final URL: https://www.google.com/robots.txt

1User-agent: *
2User-agent: Yandex
3Disallow: /search
4Allow: /search/about
5Allow: /search/howsearchworks
6Disallow: /sdch
7Disallow: /groups
8Disallow: /index.html?
9Disallow: /?
10Allow: /?hl=
11Disallow: /?hl=*&
12Allow: /?hl=*&gws_rd=ssl$
13Disallow: /?hl=*&*&gws_rd=ssl
14Allow: /?gws_rd=ssl$
15Allow: /?pt1=true$
16Disallow: /imgres
17Disallow: /u/
18Disallow: /setprefs
19Disallow: /m?
20Disallow: /m/
21Allow: /m/finance
22Disallow: /wml?
23Disallow: /wml/?
24Disallow: /wml/search?
25Disallow: /xhtml?
26Disallow: /xhtml/?
27Disallow: /xhtml/search?
28Disallow: /xml?
29Disallow: /imode?
30Disallow: /imode/?
31Disallow: /imode/search?
32Disallow: /jsky?
33Disallow: /jsky/?
34Disallow: /jsky/search?
35Disallow: /pda?
36Disallow: /pda/?
37Disallow: /pda/search?
38Disallow: /local?
39Disallow: /local_url
40Disallow: /products?
41Disallow: /product_
42Disallow: /products_
43Disallow: /products;
44Disallow: /print
45Disallow: /books/
46Disallow: /bkshp?*dq=
47Disallow: /bkshp?*q=
48Disallow: /books?*dq=
49Disallow: /books?*q=
50Disallow: /books?*qtid=
51Disallow: /books?*output=
52Disallow: /books?*pg=
53Disallow: /books?*jtp=
54Disallow: /books?*jscmd=
55Disallow: /books?*buy=
56Disallow: /books?*zoom=
57Allow: /books/about
58Allow: /books?*zoom=1
59Allow: /books?*zoom=5
60Allow: /books/content?*zoom=1
61Allow: /books/content?*zoom=5
62Disallow: /patents?
63Disallow: /patents/download/
64Disallow: /patents/pdf/
65Disallow: /patents/related/
66Disallow: /scholar
67Disallow: /citations?
68Allow: /citations?user=
69Allow: /citations?view_op=new_profile
70Allow: /citations?view_op=top_venues
71Allow: /scholar_share
72Disallow: /s?
73Disallow: /maps?
74Allow: /maps?daddr=
75Allow: /maps?entry=wc
76Allow: /maps?f=
77Allow: /maps?hl=
78Allow: /maps?q=
79Allow: /maps?saddr=
80Allow: /maps?sid=
81Allow: /maps?*output=classic
82Allow: /maps?*file=
83Disallow: /mapslt?
84Disallow: /maphp?
85Disallow: /maps/
86Allow: /maps/$
87Allow: /maps/@
88Allow: /maps/?daddr=
89Allow: /maps/?entry=wc
90Allow: /maps/?f=
91Allow: /maps/?hl=
92Allow: /maps/?q=
93Allow: /maps/?saddr=
94Allow: /maps/?sid=
95Allow: /maps/api/staticmap?
96Allow: /maps/search/
97Allow: /maps/sitemap.xml
98Allow: /maps/sitemaps/
99Allow: /maps/dir/
100Allow: /maps/d/
101Allow: /maps/reserve
102Allow: /maps/about
103Allow: /maps/contrib/
104Allow: /maps/match
105Allow: /maps/ms?
106Allow: /maps/place/
107Allow: /maps/_/
108Allow: /search?*tbm=map
109Allow: /maps/vt?
110Allow: /maps/preview
111Disallow: /maps/api/js/
112Allow: /maps/api/js
113Disallow: /mld?
114Disallow: /staticmap?
115Disallow: /help/maps/streetview/partners/welcome/
116Disallow: /help/maps/indoormaps/partners/
117Disallow: /lochp?
118Disallow: /ie?
119Disallow: /uds/
120Disallow: /transit?
121Disallow: /trends?
122Disallow: /trends/music?
123Disallow: /trends/hottrends?
124Disallow: /trends/viz?
125Disallow: /trends/embed.js?
126Disallow: /trends/fetchComponent?
127Disallow: /trends/beta
128Disallow: /trends/topics
129Disallow: /trends/explore?
130Disallow: /trends/api
131Disallow: /musica
132Disallow: /musicl
133Disallow: /musics
134Disallow: /urchin_test/
135Disallow: /movies?
136Disallow: /wapsearch?
137Disallow: /reviews/search?
138Disallow: /cbk
139Disallow: /profiles/me
140Disallow: /s2/profiles/me
141Allow: /s2/profiles
142Allow: /s2/oz
143Allow: /s2/photos
144Allow: /s2/search/social
145Allow: /s2/static
146Disallow: /s2
147Disallow: /transconsole/portal/
148Disallow: /aclk
149Disallow: /tbproxy/
150Disallow: /support/forum/search?
151Disallow: /reviews/polls/
152Disallow: /hosted/images/
153Disallow: /accounts/ClientLogin
154Disallow: /accounts/ClientAuth
155Disallow: /accounts/o8
156Allow: /accounts/o8/id
157Disallow: /quality_form?
158Disallow: /labs/popgadget/search
159Disallow: /compressiontest/
160Disallow: /analytics/feeds/
161Disallow: /analytics/partners/comments/
162Disallow: /analytics/portal/
163Disallow: /analytics/uploads/
164Allow: /alerts/manage
165Allow: /alerts/remove
166Disallow: /alerts/
167Allow: /alerts/$
168Disallow: /phone/compare/?
169Disallow: /travel/clk
170Disallow: /travel/lodging/clk
171Disallow: /travel/entity
172Disallow: /travel/search
173Disallow: /travel/flights/booking
174Disallow: /travel/flights/s/
175Disallow: /travel/flights/search
176Disallow: /travel/hotels/stories
177Disallow: /travel/hotels/*/stories
178Disallow: /travel/story
179Disallow: /hotelfinder/rpc
180Disallow: /hotels/rpc
181Disallow: /evaluation/
182Disallow: /forms/perks/
183Disallow: /shopping/suppliers/search
184Disallow: /edu/cs4hs/
185Disallow: /trustedstores/s/
186Disallow: /trustedstores/tm2
187Disallow: /trustedstores/verify
188Disallow: /shopping?
189Disallow: /shopping/product/
190Disallow: /shopping/seller
191Disallow: /shopping/ratings/account/metrics
192Disallow: /shopping/ratings/merchant/immersivedetails
193Disallow: /shopping/reviewer
194Disallow: /shopping/search
195Disallow: /shopping/deals
196Allow: /shopping?udm=28$
197Disallow: /storefront
198Disallow: /storepicker
199Disallow: /about/careers/applications/candidate-prep
200Disallow: /about/careers/applications/connect-with-a-googler
201Disallow: /about/careers/applications/jobs/results?page=
202Disallow: /about/careers/applications/jobs/results/?page=
203Disallow: /about/careers/applications/jobs/results?*&page=
204Disallow: /about/careers/applications/jobs/results/?*&page=
205Disallow: /landing/signout.html
206Disallow: /gallery/
207Disallow: /landing/now/ontap/
208Allow: /maps/reserve
209Allow: /maps/reserve/partners
210Disallow: /maps/reserve/api/
211Disallow: /maps/reserve/search
212Disallow: /maps/reserve/bookings
213Disallow: /maps/reserve/settings
214Disallow: /maps/reserve/manage
215Disallow: /maps/reserve/payment
216Disallow: /maps/reserve/receipt
217Disallow: /maps/reserve/sellersignup
218Disallow: /maps/reserve/feedback
219Disallow: /maps/reserve/terms
220Disallow: /maps/reserve/m/
221Disallow: /maps/reserve/b/
222Disallow: /maps/reserve/partner-dashboard
223Disallow: /local/cars
224Disallow: /local/dealership/
225Disallow: /local/dining/
226Disallow: /local/place/products/
227Disallow: /local/place/reviews/
228Disallow: /local/place/rap/
229Disallow: /local/tab/
230Disallow: /localservices/
231Allow: /localservices/coupon/
232Disallow: /nonprofits/account/
233Disallow: /uviewer
234Disallow: /landing/cmsnext-root/
235
236# AdsBot
237User-agent: AdsBot-Google
238Disallow: /maps/api/js/
239Allow: /maps/api/js
240Disallow: /maps/api/place/js/
241Disallow: /maps/api/staticmap
242Disallow: /maps/api/streetview
243
244# New user agent groups must also have a user agent reference in the global (*)
245# group. See "Order of precedence" section in
246# https://goo.gle/rep#order-of-precedence-for-user-agents
247User-agent: Yandex
248Disallow: /about/careers/applications/jobs/results
249Disallow: /about/careers/applications-a/jobs/results
250
251# Crawlers of certain social media sites are allowed to access page markup when
252# google.com/imgres* links are shared. To learn more, please contact
253# [email protected].
254User-agent: facebookexternalhit
255User-agent: Twitterbot
256Allow: /imgres
257Allow: /search
258Disallow: /groups
259Disallow: /hosted/images/
260Disallow: /m/
261
262Sitemap: https://www.google.com/sitemap.xml