NeuralCrawl

Google / robots.txt snapshot

← back to google.com · fetched 2026-07-24T02:19:07Z (13d ago) · HTTP 200 · 6624 bytes · sha256 1aba570a1b32deb6 · raw

final URL: https://www.google.com/robots.txt

1User-agent: *
2User-agent: Yandex
3Disallow: /search
4Allow: /search/about
5Allow: /search/howsearchworks
6Disallow: /sdch
7Disallow: /groups
8Disallow: /index.html?
9Disallow: /?
10Disallow: /goto?
11Allow: /?hl=
12Disallow: /?hl=*&
13Allow: /?hl=*&gws_rd=ssl$
14Disallow: /?hl=*&*&gws_rd=ssl
15Allow: /?gws_rd=ssl$
16Allow: /?pt1=true$
17Disallow: /imgres
18Disallow: /u/
19Disallow: /setprefs
20Disallow: /m?
21Disallow: /m/
22Allow: /m/finance
23Disallow: /wml?
24Disallow: /wml/?
25Disallow: /wml/search?
26Disallow: /xhtml?
27Disallow: /xhtml/?
28Disallow: /xhtml/search?
29Disallow: /xml?
30Disallow: /imode?
31Disallow: /imode/?
32Disallow: /imode/search?
33Disallow: /jsky?
34Disallow: /jsky/?
35Disallow: /jsky/search?
36Disallow: /pda?
37Disallow: /pda/?
38Disallow: /pda/search?
39Disallow: /local?
40Disallow: /local_url
41Disallow: /products?
42Disallow: /product_
43Disallow: /products_
44Disallow: /products;
45Disallow: /print
46Disallow: /books/
47Disallow: /bkshp?*dq=
48Disallow: /bkshp?*q=
49Disallow: /books?*dq=
50Disallow: /books?*q=
51Disallow: /books?*qtid=
52Disallow: /books?*output=
53Disallow: /books?*pg=
54Disallow: /books?*jtp=
55Disallow: /books?*jscmd=
56Disallow: /books?*buy=
57Disallow: /books?*zoom=
58Allow: /books/about
59Allow: /books?*zoom=1
60Allow: /books?*zoom=5
61Allow: /books/content?*zoom=1
62Allow: /books/content?*zoom=5
63Disallow: /patents?
64Disallow: /patents/download/
65Disallow: /patents/pdf/
66Disallow: /patents/related/
67Disallow: /scholar
68Disallow: /citations?
69Allow: /citations?user=
70Allow: /citations?view_op=new_profile
71Allow: /citations?view_op=top_venues
72Allow: /scholar_share
73Disallow: /s?
74Disallow: /maps?
75Allow: /maps?daddr=
76Allow: /maps?entry=wc
77Allow: /maps?f=
78Allow: /maps?hl=
79Allow: /maps?q=
80Allow: /maps?saddr=
81Allow: /maps?sid=
82Allow: /maps?*output=classic
83Allow: /maps?*file=
84Disallow: /mapslt?
85Disallow: /maphp?
86Disallow: /maps/
87Allow: /maps/$
88Allow: /maps/@
89Allow: /maps/?daddr=
90Allow: /maps/?entry=wc
91Allow: /maps/?f=
92Allow: /maps/?hl=
93Allow: /maps/?q=
94Allow: /maps/?saddr=
95Allow: /maps/?sid=
96Allow: /maps/api/staticmap?
97Allow: /maps/search/
98Allow: /maps/sitemap.xml
99Allow: /maps/sitemaps/
100Allow: /maps/dir/
101Allow: /maps/d/
102Allow: /maps/reserve
103Allow: /maps/about
104Allow: /maps/contrib/
105Allow: /maps/match
106Allow: /maps/ms?
107Allow: /maps/place/
108Allow: /maps/_/
109Allow: /search?*tbm=map
110Allow: /maps/vt?
111Allow: /maps/preview
112Disallow: /maps/api/js/
113Allow: /maps/api/js
114Disallow: /mld?
115Disallow: /staticmap?
116Disallow: /help/maps/streetview/partners/welcome/
117Disallow: /help/maps/indoormaps/partners/
118Disallow: /lochp?
119Disallow: /ie?
120Disallow: /uds/
121Disallow: /transit?
122Disallow: /trends?
123Disallow: /trends/music?
124Disallow: /trends/hottrends?
125Disallow: /trends/viz?
126Disallow: /trends/embed.js?
127Disallow: /trends/fetchComponent?
128Disallow: /trends/beta
129Disallow: /trends/topics
130Disallow: /trends/explore?
131Disallow: /trends/api
132Disallow: /musica
133Disallow: /musicl
134Disallow: /musics
135Disallow: /urchin_test/
136Disallow: /movies?
137Disallow: /wapsearch?
138Disallow: /reviews/search?
139Disallow: /cbk
140Disallow: /profiles/me
141Disallow: /s2/profiles/me
142Allow: /s2/profiles
143Allow: /s2/oz
144Allow: /s2/photos
145Allow: /s2/search/social
146Allow: /s2/static
147Disallow: /s2
148Disallow: /transconsole/portal/
149Disallow: /aclk
150Disallow: /tbproxy/
151Disallow: /support/forum/search?
152Disallow: /reviews/polls/
153Disallow: /hosted/images/
154Disallow: /accounts/ClientLogin
155Disallow: /accounts/ClientAuth
156Disallow: /accounts/o8
157Allow: /accounts/o8/id
158Disallow: /quality_form?
159Disallow: /labs/popgadget/search
160Disallow: /compressiontest/
161Disallow: /analytics/feeds/
162Disallow: /analytics/partners/comments/
163Disallow: /analytics/portal/
164Disallow: /analytics/uploads/
165Allow: /alerts/manage
166Allow: /alerts/remove
167Disallow: /alerts/
168Allow: /alerts/$
169Disallow: /phone/compare/?
170Disallow: /travel/clk
171Disallow: /travel/lodging/clk
172Disallow: /travel/entity
173Disallow: /travel/search
174Disallow: /travel/flights/booking
175Disallow: /travel/flights/s/
176Disallow: /travel/flights/search
177Disallow: /travel/hotels/stories
178Disallow: /travel/hotels/*/stories
179Disallow: /travel/story
180Disallow: /hotelfinder/rpc
181Disallow: /hotels/rpc
182Disallow: /evaluation/
183Disallow: /forms/perks/
184Disallow: /shopping/suppliers/search
185Disallow: /edu/cs4hs/
186Disallow: /trustedstores/s/
187Disallow: /trustedstores/tm2
188Disallow: /trustedstores/verify
189Disallow: /shopping?
190Disallow: /shopping/product/
191Disallow: /shopping/seller
192Disallow: /shopping/ratings/account/metrics
193Disallow: /shopping/ratings/merchant/immersivedetails
194Disallow: /shopping/reviewer
195Disallow: /shopping/search
196Disallow: /shopping/deals
197Allow: /shopping?udm=28$
198Disallow: /storefront
199Disallow: /storepicker
200Disallow: /about/careers/applications/candidate-prep
201Disallow: /about/careers/applications/connect-with-a-googler
202Disallow: /about/careers/applications/jobs/results?page=
203Disallow: /about/careers/applications/jobs/results/?page=
204Disallow: /about/careers/applications/jobs/results?*&page=
205Disallow: /about/careers/applications/jobs/results/?*&page=
206Disallow: /landing/signout.html
207Disallow: /gallery/
208Disallow: /landing/now/ontap/
209Allow: /maps/reserve
210Allow: /maps/reserve/partners
211Disallow: /maps/reserve/api/
212Disallow: /maps/reserve/search
213Disallow: /maps/reserve/bookings
214Disallow: /maps/reserve/settings
215Disallow: /maps/reserve/manage
216Disallow: /maps/reserve/payment
217Disallow: /maps/reserve/receipt
218Disallow: /maps/reserve/sellersignup
219Disallow: /maps/reserve/feedback
220Disallow: /maps/reserve/terms
221Disallow: /maps/reserve/m/
222Disallow: /maps/reserve/b/
223Disallow: /maps/reserve/partner-dashboard
224Disallow: /local/cars
225Disallow: /local/dealership/
226Disallow: /local/dining/
227Disallow: /local/place/products/
228Disallow: /local/place/reviews/
229Disallow: /local/place/rap/
230Disallow: /local/tab/
231Disallow: /localservices/
232Allow: /localservices/coupon/
233Disallow: /nonprofits/account/
234Disallow: /uviewer
235Disallow: /landing/cmsnext-root/
236
237# AdsBot
238User-agent: AdsBot-Google
239Disallow: /maps/api/js/
240Allow: /maps/api/js
241Disallow: /maps/api/place/js/
242Disallow: /maps/api/staticmap
243Disallow: /maps/api/streetview
244
245# New user agent groups must also have a user agent reference in the global (*)
246# group. See "Order of precedence" section in
247# https://goo.gle/rep#order-of-precedence-for-user-agents
248User-agent: Yandex
249Disallow: /about/careers/applications/jobs/results
250Disallow: /about/careers/applications-a/jobs/results
251
252# Crawlers of certain social media sites are allowed to access page markup when
253# google.com/imgres* links are shared. To learn more, please contact
254# [email protected].
255User-agent: facebookexternalhit
256User-agent: Twitterbot
257Allow: /imgres
258Allow: /search
259Disallow: /groups
260Disallow: /hosted/images/
261Disallow: /m/
262
263Sitemap: https://www.google.com/sitemap.xml