NeuralCrawl

Amazon / robots.txt snapshot

← back to amazon.com · fetched 2026-06-20T01:10:30Z (1mo ago) · HTTP 200 · 7888 bytes · sha256 9245e23a64e506a5 · raw

final URL: https://www.amazon.com/robots.txt

1User-agent: *
2Disallow: /exec/obidos/account-access-login
3Disallow: /exec/obidos/change-style
4Disallow: /exec/obidos/flex-sign-in
5Disallow: /exec/obidos/handle-buy-box
6Disallow: /exec/obidos/tg/cm/member/
7Disallow: /gp/aw/help/id=sss
8Disallow: /gp/cart
9Disallow: /gp/flex
10Disallow: /gp/product/e-mail-friend
11Disallow: /gp/product/product-availability
12Disallow: /gp/product/rate-this-item
13Disallow: /gp/sign-in
14Disallow: /gp/reader
15Disallow: /gp/sitbv3/reader
16Disallow: /gp/richpub/syltguides/create
17Disallow: /gp/gfix
18Disallow: /gp/associations/wizard.html
19Disallow: /gp/dmusic/order
20Disallow: /gp/legacy-handle-buy-box.html
21Disallow: /gp/aws/ssop
22Disallow: /gp/yourstore
23Disallow: /gp/gift-central/organizer/add-wishlist
24Disallow: /gp/vote
25Disallow: /gp/voting/
26Disallow: /gp/music/wma-pop-up
27Disallow: /gp/customer-images
28Disallow: /gp/richpub/listmania/createpipeline
29Disallow: /gp/content-form
30Disallow: /gp/pdp/invitation/invite
31Disallow: /gp/customer-reviews/common/du
32Disallow: /gp/customer-reviews/write-a-review.html
33Disallow: /gp/associations/wizard.html
34Disallow: /gp/music/clipserve
35Disallow: /gp/customer-media/upload
36Disallow: /gp/history
37Disallow: /gp/item-dispatch
38Disallow: /gp/dmusic/order/handle-buy-box.html
39Disallow: /gp/recsradio
40Disallow: /gp/slredirect
41Disallow: /dp/shipping/
42Disallow: /dp/twister-update/
43Disallow: /dp/manual-submit/
44Disallow: /dp/e-mail-friend/
45Disallow: /dp/product-availability/
46Disallow: /dp/rate-this-item/
47Disallow: /gp/registry/wishlist/*/reserve
48Disallow: /gp/structured-ratings/actions/get-experience.html
49Disallow: /gp/twitter/
50Disallow: /ap/signin
51Disallow: /gp/registry/wishlist/
52Disallow: /wishlist/
53Allow: /wishlist/universal*
54Allow: /wishlist/vendor-button*
55Allow: /wishlist/get-button*
56Disallow: /gp/wishlist/
57Allow: /gp/wishlist/universal*
58Allow: /gp/wishlist/vendor-button*
59Allow: /gp/wishlist/ipad-install*
60Disallow: /registry/wishlist/
61Disallow: /review/common/du
62Disallow: /gp/registry/search.html
63Disallow: /product-reviews/B0069IY63Y
64Disallow: /gp/orc/rml/
65Disallow: */gcrnsts
66Disallow: /gp/gc/widget
67Disallow: /gp/dmusic/mp3/player
68Disallow: /gp/entity-alert/external
69Disallow: /gp/customer-reviews/dynamic/sims-box
70Disallow: /review/dynamic/sims-box
71Disallow: /gp/redirect.html
72Disallow: /gp/twister/ajaxv2
73Disallow: /ss/twister/ajax
74Disallow: /b?*node=7454917011
75Disallow: /b?*node=7454927011
76Disallow: /b?*node=7454939011
77Disallow: /b?*node=7454898011
78Disallow: /gp/customer-media/actions/delete/
79Disallow: /gp/customer-media/actions/edit-caption/
80Disallow: /gp/dmusic/
81Allow: /gp/dmusic/promotions/PrimeMusic
82Allow: /gp/dmusic/promotions/AmazonMusicUnlimited
83Disallow: /gp/offer-listing/
84Disallow: /b?*node=9052533011
85Disallow: /lm/R1XIHQVKXSKBNJ
86Disallow: /lm/R3HQ5WJSZK6QSO
87Disallow: /surprise/
88Disallow: /local/ajax/
89Disallow: /gp/socialmedia/giveaways
90Disallow: /gp/b2b-rd
91Disallow: /gp/aw/so.html
92Disallow: /reviews/iframe
93Disallow: /gp/switch-language
94Disallow: /ga/p/
95Disallow: /gp/profile/
96Disallow: /giveaway/host/setup/
97Disallow: /ss/customer-reviews/lighthouse/
98Disallow: /ospublishing/story/*
99Disallow: /gp/aw/ol/
100Disallow: /gp/promotion/
101Disallow: /hz/leaderboard/top-reviewers/
102Disallow: /creatorhub
103Disallow: /creatorhub/*
104Disallow: /slp/s$
105Disallow: /-/
106Allow: /-/es/
107Allow: /-/en$
108Allow: /-/zh_TW/
109Allow: /-/zh_TW$
110Allow: /-/he/
111Allow: /-/he$
112Disallow: /hz/help/contact/*/message/$
113Disallow: /gp/aw/shoppingAids/
114Disallow: /rss/people/*/reviews
115Disallow: /gp/pdp/rss/*/reviews
116Disallow: /gp/cdp/member-reviews/
117Disallow: /gp/aw/cr/
118Allow: /gp/offer-listing/B000
119Allow: /gp/offer-listing/9000
120Disallow: /gp/aag
121Allow: /gp/aag/main?*seller=ABVFEJU8LS620
122Disallow: /gp/pdp/profile/
123Disallow: /gp/help/customer/express/c2c/
124Disallow: /slp/*/b$
125Disallow: /hz/contact-us/ajax/initiate-trusted-contact/
126Disallow: /hp/video/
127Disallow: /gp/video/api
128Disallow: /gp/video/settings
129Disallow: /gp/video/library
130Disallow: /gp/video/watchlist
131Disallow: /gp/video/mystuff
132Disallow: /gp/video/profiles
133Disallow: /gp/video/search
134Disallow: /gp/video/auth
135Disallow: /gp/video/salp/i
136Disallow: /icdp/
137
138User-agent: EtaoSpider
139Disallow: /
140
141User-agent: GPTBot
142Disallow: /
143
144
145User-agent: CCBot
146Disallow: /
147
148User-Agent: PerplexityBot
149Disallow: /
150
151User-agent: Google-Extended
152Disallow: /
153
154User-agent: GoogleAgent-Mariner
155Disallow: /
156
157User-agent: GoogleAgent-Shopping
158Disallow: /
159
160User-agent: ClaudeBot
161Disallow: /
162
163User-agent: meta-externalagent
164Disallow: /
165
166User-agent: Bytespider
167Disallow: /
168
169User-agent: Scrapy
170Disallow: /
171
172User-agent: PetalBot
173Disallow: /
174
175User-agent: Devin
176Disallow: /
177
178User-agent: omgili
179Disallow: /
180
181User-agent: AI2Bot
182Disallow: /
183
184User-agent: Gemini-Deep-Research
185Disallow: /
186
187User-agent: PanguBot
188Disallow: /
189
190User-agent: MistralAI-User
191Disallow: /
192
193User-agent: Diffbot
194Disallow: /
195
196User-agent: Sidetrade indexer bot
197Disallow: /
198
199User-agent: Google-NotebookLM
200Disallow: /
201
202User-agent: DuckAssistBot
203Disallow: /
204
205User-agent: Copilot
206Disallow: /
207
208User-agent: Ai2Bot-Dolma
209Disallow: /
210
211User-agent: Google-CloudVertexBot
212Disallow: /
213
214User-agent: meta-externalfetcher
215Disallow: /
216
217User-agent: iaskspider
218Disallow: /
219
220User-agent: wpbot
221Disallow: /
222
223User-agent: aiHitBot
224Disallow: /
225
226User-agent: cohere-ai
227Disallow: /
228
229User-agent: FriendlyCrawler
230Disallow: /
231
232User-agent: img2dataset
233Disallow: /
234
235User-agent: VelenPublicWebCrawler
236Disallow: /
237
238User-agent: YouBot
239Disallow: /
240
241User-agent: Brightbot
242Disallow: /
243
244User-agent: Claude-User
245Disallow: /
246
247User-agent: ClaudeBot
248Disallow: /
249
250User-agent: Claude-SearchBot
251Disallow: /
252
253User-agent: Perplexity-User
254Disallow: /
255
256User-agent: ISSCyberRiskCrawler
257Disallow: /
258
259User-agent: CopilotNative
260Disallow: /
261
262User-agent: CopilotSapphire
263Disallow: /
264
265User-agent: imgproxy
266Disallow: /
267
268User-agent: SemrushBot-SWA
269Disallow: /
270
271User-agent: ChatGPT-User
272Disallow: /
273
274User-agent: OAI-SearchBot
275Disallow: /
276
277User-agent: Timpibot
278Disallow: /
279
280User-agent: Thinkbot
281Disallow: /
282
283User-agent: AddSearchBot
284Disallow: /
285
286User-agent: AI2Bot-DeepResearchEval
287Disallow: /
288
289User-agent: Andibot
290Disallow: /
291
292User-agent: Anomura
293Disallow: /
294
295User-agent: atlassian-bot
296Disallow: /
297
298User-agent: BuddyBot
299Disallow: /
300
301User-agent: Channel3Bot
302Disallow: /
303
304User-agent: ChatGLM-Spider
305Disallow: /
306
307User-agent: Claude-Web
308Disallow: /
309
310User-agent: Cloudflare-AutoRAG
311Disallow: /
312
313User-agent: CloudflareBrowserRenderingCrawler
314Disallow: /
315
316User-agent: cohere-training-data-crawler
317Disallow: /
318
319User-agent: Cotoyogi
320Disallow: /
321
322User-agent: Crawl4AI
323Disallow: /
324
325User-agent: Datenbank Crawler
326Disallow: /
327
328User-agent: DeepSeekBot
329Disallow: /
330
331User-agent: Echobot Bot
332Disallow: /
333
334User-agent: EchoboxBot
335Disallow: /
336
337User-agent: factset_spyderbot
338Disallow: /
339
340User-agent: Google-Firebase
341Disallow: /
342
343User-agent: GoogleOther
344Disallow: /
345
346User-agent: Grok-DeepSearch
347Disallow: /
348
349User-agent: GrokBot
350Disallow: /
351
352User-agent: iAskBot
353Disallow: /
354
355User-agent: IbouBot
356Disallow: /
357
358User-agent: ICC-Crawler
359Disallow: /
360
361User-agent: ImagesiftBot
362Disallow: /
363
364User-agent: imageSpider
365Disallow: /
366
367User-agent: Kangaroo Bot
368Disallow: /
369
370User-agent: KlaviyoAIBot
371Disallow: /
372
373User-agent: KunatoCrawler
374Disallow: /
375
376User-agent: laion-huggingface-processor
377Disallow: /
378
379User-agent: Linguee Bot
380Disallow: /
381
382User-agent: LinkupBot
383Disallow: /
384
385User-agent: Manus-User
386Disallow: /
387
388User-agent: meta-webindexer
389Disallow: /
390
391User-agent: netEstate Imprint Crawler
392Disallow: /
393
394User-agent: Panscient
395Disallow: /
396
397User-agent: panscient.com
398Disallow: /
399
400User-agent: PhindBot
401Disallow: /
402
403User-agent: Poggio-Citations
404Disallow: /
405
406User-agent: QualifiedBot
407Disallow: /
408
409User-agent: SBIntuitionsBot
410Disallow: /
411
412User-agent: ShapBot
413Disallow: /
414
415User-agent: TavilyBot
416Disallow: /
417
418User-agent: TerraCotta
419Disallow: /
420
421User-agent: TwinAgent
422Disallow: /
423
424User-agent: webzio-extended
425Disallow: /
426
427User-agent: WRTNBot
428Disallow: /
429
430User-agent: xAI-Grok
431Disallow: /
432
433User-agent: YaK
434Disallow: /
435
436User-agent: ZanistaBot
437Disallow: /