NeuralCrawl

Best Buy / robots.txt snapshot

← back to bestbuy.com · fetched 2026-08-24T19:21:08Z (25d ago) · HTTP 200 · 14897 bytes · sha256 fa6d343f941e71e9 · raw

final URL: https://www.bestbuy.com/robots.txt

1# --------------------------------------------------
2# BEST BUY OFFICIAL ENTERPRISE ROBOTS.TXT
3# Optimized for search engines, AI discovery, ads, and crawl control
4# --------------------------------------------------
5
6# ==================================================
7# BUCKET 1: AI SEARCH & DISCOVERY BOT ACCESS
8# Allows conversational answer engines to securely access clean landing pages.
9# Mirrors explicit production parameters to protect long-tail revenue.
10# ==================================================
11User-agent: OAI-SearchBot
12User-agent: ChatGPT-User
13User-agent: Perplexity-User
14User-agent: GPTBot
15User-agent: Claude-User
16User-agent: Claudebot
17User-agent: PerplexityBot
18User-agent: Applebot
19User-agent: Google-Extended
20User-agent: Gemeni-AI
21User-agent: Gemini-Deep-Research
22User-agent: AmazonBot
23
24Allow: /site/
25Allow: /discover-learn/
26
27# System Application Blockouts
28Disallow: /nex/
29Disallow: /m/e/*
30Disallow: /browse-api/*
31Disallow: /api/*.json
32Disallow: /availability/*
33Disallow: /cart
34Disallow: /cart/*
35Disallow: /cartitems/*
36Disallow: /fulfillment/*
37Disallow: /location/*
38Disallow: /pricing/*
39Disallow: /productfulfillment/*
40Disallow: /profile/*
41Disallow: /services/plans/*
42Disallow: /services/shoppingscheduler*
43Disallow: /site/builder/*
44Disallow: /site/modals/*
45Disallow: /site/dni/*
46Disallow: /site/offerbuilder/*
47
48# User Dynamic Input Paths
49Disallow: /site/questions/submission*
50Disallow: /site/answers/submission*
51Disallow: /site/reviews/submission*
52
53# Production-Safe Explicit Tab Fragments
54Disallow: /*template=_gameDetailsTab
55Disallow: /*template=_movieDetailsTab
56Disallow: /*template=_musicDetailsTab
57Disallow: /*template=_softwareDetailsTab
58Disallow: /*template=_accessoriesTab
59Disallow: /*template=_castAndCrewTab
60Disallow: /*template=_editorialTab
61Disallow: /*template=_episodesTab
62Disallow: /*template=_protectionAndServicesTab
63Disallow: /*template=_specificationsTab
64Disallow: /*template=_featuresSpoke
65Disallow: /*template=_buyingOptionsNewTab
66
67# Production-Safe Explicit Core Facet Controls (Brand/Category left open)
68Disallow: *armodelformat_facet*
69Disallow: *awardtype_facet*
70Disallow: *awardwinner_facet*
71Disallow: *awardyear_facet*
72Disallow: *blenderfunctions_facet*
73Disallow: *boxdesign_facet*
74Disallow: *cast_facet*
75Disallow: *cleaningpathwidthrange_facet*
76Disallow: *consumerlifestage_facet*
77Disallow: *decade_facet*
78Disallow: *defrosttype_facet*
79Disallow: *departmentid_facet*
80Disallow: *depthtype_facet*
81Disallow: *displayport_facet*
82Disallow: *doorstylesv_facet*
83Disallow: *drawertype_facet*
84Disallow: *dynamiccategory_facet*
85Disallow: *effectivemegapixels_facet*
86Disallow: *flashmodesmv_facet*
87Disallow: *framematerialmv_facet*
88Disallow: *giftcardthemesv_facet*
89Disallow: *handletype_facet*
90Disallow: *headsetstylesv_facet*
91Disallow: *heatingsource_facet*
92Disallow: *imagesensorsize_facet*
93Disallow: *imagesensortype11_facet*
94Disallow: *includedcolors_facet*
95Disallow: *inkmodelcompatibilitymv_facet*
96Disallow: *internalorexternal_facet*
97Disallow: *laptopuse_facet*
98Disallow: *lengthofcable_facet*
99Disallow: *mattressmaterial_facet*
100Disallow: *maximumsensitivity_facet*
101Disallow: *minfocallengthrange_facet*
102Disallow: *mobileinternalmemory_facet*
103Disallow: *monitorresponsetimerange_facet*
104Disallow: *monitorsnumberofusb30typecports_facet*
105Disallow: *monitorsnumberofusb31typecports_facet*
106Disallow: *mountmaterial_facet*
107Disallow: *networkingcabletype_facet*
108Disallow: *numberofbedroomsef_facet*
109Disallow: *numberofburnerssv_facet*
110Disallow: *numberofbuttons_facet*
111Disallow: *numberofdryingcycles_facet*
112Disallow: *numberoffanspeedssv_facet*
113Disallow: *numberofhdmiinputssv_facet*
114Disallow: *numberoflanports_facet*
115Disallow: *numberofspeakers_facet*
116Disallow: *numberofspeedssv_facet*
117Disallow: *numberofthunderbolt_ports_facet*
118Disallow: *numberofwashrinsecycle_facet*
119Disallow: *papersize_facet*
120Disallow: *playbackformatsmv_facet*
121Disallow: *productcategories_facet*
122Disallow: *screensizediagonalrange_facet*
123Disallow: *securityconnectivity_facet*
124Disallow: *securityresolution_facet*
125Disallow: *securitystoragetype_facet*
126Disallow: *softwareformat_facet*
127Disallow: *sqbprocessormodel_facet*
128Disallow: *sqbtotalstoragecapacity_facet*
129Disallow: *stringtype_facet*
130Disallow: *subclassid_facet*
131Disallow: *subgenre_facet*
132Disallow: *thruthedoordispenser_facet*
133Disallow: *toastingslotssv_facet*
134Disallow: *totaltechdeals_facet*
135Disallow: *touchscreentype_facet*
136Disallow: *transferrate_facet*
137Disallow: *transistorsize_facet*
138Disallow: *transmitterdistance_facet*
139Disallow: *treeheight_facet*
140Disallow: *treelightbulbcolor_facet*
141Disallow: *tubfinishmv_facet*
142Disallow: *varyinganglescreen_facet*
143Disallow: *vendorcollection_facet*
144Disallow: *videomemory_facet*
145Disallow: *viewfindertypemv_facet*
146Disallow: *virtualrealitycompatibility_facet*
147Disallow: *wanportssv_facet*
148Disallow: *wearingstyle_facet*
149Disallow: *currentprice2_facet*
150Disallow: *customerreviews_facet*Customer*Rating*5*
151Disallow: *customerreviews_facet*Customer*Rating*4*
152Disallow: *customerreviews_facet*Customer*Rating*3*
153Disallow: *customerreviews_facet*Customer*Rating*2*
154Disallow: *customerreviews_facet*Customer*Rating*1*
155Disallow: *soldout_facet*
156Disallow: */site/promo/*?*_facet*
157Disallow: */product/list/*
158Disallow: */awacs-ingestor/*
159
160# Exact Pagination Safeguards
161Disallow: *cp=1*
162Disallow: *cp=20*
163Disallow: *cp=6*
164Disallow: *cp=7*
165Disallow: *cp=8*
166Disallow: *cp=9*
167Disallow: *?*page=*&page=*$
168
169# Search Parameters Controls
170Disallow: */site/searchpage.jsp?st=*
171Disallow: *?id*_facet*&st=*
172Disallow: *?id*_facet*_facet*&st=*
173Disallow: *?cp=*&id=*&st=*
174
175# Malformed URL Traps (Universal Protection)
176Disallow: *==$
177Disallow: */-?
178Disallow: */undefined$
179Disallow: */null$
180Disallow: *javascript:winopengld*
181
182# ==================================================
183# BUCKET 2: AI TRAINING / DATA INGESTION FIREWALL
184# Completely blocks models from scraping copy assets for foundational training.
185# ==================================================
186User-agent: GPTBot-Web
187User-agent: CCBot
188User-agent: Anthropic-AI
189User-agent: Claude-Web
190User-agent: Bytespider
191Disallow: /[1.1]
192
193# ==================================================
194# BUCKET 3: UNIFIED WEB ENGINES & UNIVERSAL DEFAULTS
195# Consolidated grouping ensures zero functional divergence between indexing bots.
196# ==================================================
197User-agent: Googlebot
198User-agent: Bingbot
199User-agent: DuckDuckBot
200User-agent: Slurp
201User-agent: *
202
203# Standard Asset Pathways
204Allow: /~assets/
205Allow: /discover-learn/
206
207# Core System Blockouts
208Disallow: /nex/
209Disallow: /m/e/*
210Disallow: /site/builder/*
211Disallow: /browse-api/*
212Disallow: /api/*.json
213Disallow: /availability/*
214Disallow: /cart
215Disallow: /cart/*
216Disallow: /cartitems/*
217Disallow: /fulfillment/*
218Disallow: /location/*
219Disallow: /pricing/*
220Disallow: /productfulfillment/*
221Disallow: /profile/*
222Disallow: /site/modals/*
223Disallow: /services/plans/*
224Disallow: /site/dni/*
225Disallow: /site/offerbuilder/*
226Disallow: /services/shoppingscheduler*
227
228# Staging, Internal Test, & Promotional Exclusion Matrices
229Disallow: /site/clp/test-page/*
230Disallow: /site/test-page/*
231Disallow: /site/misc/test-pages/*
232Disallow: /site/offer/*
233Disallow: /site/compare?skus=*
234Disallow: /site/clp/conditional-offers/*
235Disallow: /site/total-tech-support/*
236Disallow: /site/clp/annette-test-listing/*
237Disallow: /site/shop-view-test-page/*
238Disallow: /site/clp/test/*
239Disallow: /site/test-cloud-ready/*
240Disallow: /site/clp/mystery-rewards-certificate/pcmcat1524134875052.c?id=pcmcat1524134875052
241Disallow: /purchasehistory/gift-receipt*
242Disallow: /contact-us?*
243Disallow: /ease/email
244Disallow: /ihc/invoice*
245Disallow: /identity/newAccount?*
246Disallow: /identity/signin?*
247Disallow: /identity/signin/forgotPassword?*
248Disallow: /dossier/service/appointment/*/*-*
249Disallow: /dossier/service/appointment/search?*
250Disallow: */product-selector/*
251Disallow: */email-subscription/*
252
253# Legacy Template Controller Protections (Parameter Strictness Maintained)
254Disallow: /site/olspage.jsp?id=pcat17098*
255Disallow: /site/olstemplatemapper.jsp?id=pcat17098*
256Disallow: /site/olspage.jsp?id=pcat17089*
257Disallow: /site/olstemplatemapper.jsp?id=pcat17089*
258Disallow: /site/olspage.jsp?id=cat13503*
259Disallow: /site/olstemplatemapper.jsp?id=cat13503*
260Disallow: /site/olspage.jsp?id=pcat17007*
261Disallow: /site/olstemplatemapper.jsp?id=pcat17007*
262Disallow: /site/olspage.jsp?id=pcat17408*
263Disallow: /site/olstemplatemapper.jsp?id=pcat17408*
264Disallow: /site/olspage.jsp?id=cat12091*
265Disallow: /site/olstemplatemapper.jsp?id=cat12091*
266Disallow: /site/olspage.jsp?id=pcat17005*
267Disallow: /site/olstemplatemapper.jsp?id=pcat17005*
268Disallow: /site/olspage.jsp?id=cat12076*
269Disallow: /site/olstemplatemapper.jsp?id=cat12076*
270Disallow: /site/olspage.jsp?id=cat12080*
271Disallow: /site/olstemplatemapper.jsp?id=cat12080*
272Disallow: /site/olspage.jsp?id=cat12101*
273Disallow: /site/olstemplatemapper.jsp?id=cat12101*
274Disallow: /site/olstemplatemapper.jsp?id=pcat17006*
275Disallow: /site/olspage.jsp?id=pcat17006*
276Disallow: */site/olspage.jsp*
277
278# Asynchronous Layout Fragment Controls
279Disallow: /*template=_gameDetailsTab
280Disallow: /*template=_movieDetailsTab
281Disallow: /*template=_musicDetailsTab
282Disallow: /*template=_softwareDetailsTab
283Disallow: /*template=_accessoriesTab
284Disallow: /*template=_castAndCrewTab
285Disallow: /*template=_editorialTab
286Disallow: /*template=_episodesTab
287Disallow: /*template=_protectionAndServicesTab
288Disallow: /*template=_specificationsTab
289Disallow: /*template=_featuresSpoke
290Disallow: /*template=_buyingOptionsNewTab
291
292# User Input Paths
293Disallow: /site/questions/submission*
294Disallow: /site/answers/submission*
295Disallow: /site/reviews/submission*
296
297# Granular Facet Exclusions
298Disallow: *armodelformat_facet*
299Disallow: *awardtype_facet*
300Disallow: *awardwinner_facet*
301Disallow: *awardyear_facet*
302Disallow: *blenderfunctions_facet*
303Disallow: *boxdesign_facet*
304Disallow: *cast_facet*
305Disallow: *cleaningpathwidthrange_facet*
306Disallow: *consumerlifestage_facet*
307Disallow: *decade_facet*
308Disallow: *defrosttype_facet*
309Disallow: *departmentid_facet*
310Disallow: *depthtype_facet*
311Disallow: *displayport_facet*
312Disallow: *doorstylesv_facet*
313Disallow: *drawertype_facet*
314Disallow: *dynamiccategory_facet*
315Disallow: *effectivemegapixels_facet*
316Disallow: *flashmodesmv_facet*
317Disallow: *framematerialmv_facet*
318Disallow: *giftcardthemesv_facet*
319Disallow: *handletype_facet*
320Disallow: *headsetstylesv_facet*
321Disallow: *heatingsource_facet*
322Disallow: *imagesensorsize_facet*
323Disallow: *imagesensortype11_facet*
324Disallow: *includedcolors_facet*
325Disallow: *inkmodelcompatibilitymv_facet*
326Disallow: *internalorexternal_facet*
327Disallow: *laptopuse_facet*
328Disallow: *lengthofcable_facet*
329Disallow: *mattressmaterial_facet*
330Disallow: *maximumsensitivity_facet*
331Disallow: *minfocallengthrange_facet*
332Disallow: *mobileinternalmemory_facet*
333Disallow: *monitorresponsetimerange_facet*
334Disallow: *monitorsnumberofusb30typecports_facet*
335Disallow: *monitorsnumberofusb31typecports_facet*
336Disallow: *mountmaterial_facet*
337Disallow: *networkingcabletype_facet*
338Disallow: *numberofbedroomsef_facet*
339Disallow: *numberofburnerssv_facet*
340Disallow: *numberofbuttons_facet*
341Disallow: *numberofdryingcycles_facet*
342Disallow: *numberoffanspeedssv_facet*
343Disallow: *numberofhdmiinputssv_facet*
344Disallow: *numberoflanports_facet*
345Disallow: *numberofspeakers_facet*
346Disallow: *numberofspeedssv_facet*
347Disallow: *numberofthunderbolt_ports_facet*
348Disallow: *numberofwashrinsecycle_facet*
349Disallow: *papersize_facet*
350Disallow: *playbackformatsmv_facet*
351Disallow: *productcategories_facet*
352Disallow: *screensizediagonalrange_facet*
353Disallow: *securityconnectivity_facet*
354Disallow: *securityresolution_facet*
355Disallow: *securitystoragetype_facet*
356Disallow: *softwareformat_facet*
357Disallow: *sqbprocessormodel_facet*
358Disallow: *sqbtotalstoragecapacity_facet*
359Disallow: *stringtype_facet*
360Disallow: *subclassid_facet*
361Disallow: *subgenre_facet*
362Disallow: *thruthedoordispenser_facet*
363Disallow: *toastingslotssv_facet*
364Disallow: *totaltechdeals_facet*
365Disallow: *touchscreentype_facet*
366Disallow: *transferrate_facet*
367Disallow: *transistorsize_facet*
368Disallow: *transmitterdistance_facet*
369Disallow: *treeheight_facet*
370Disallow: *treelightbulbcolor_facet*
371Disallow: *tubfinishmv_facet*
372Disallow: *varyinganglescreen_facet*
373Disallow: *vendorcollection_facet*
374Disallow: *videomemory_facet*
375Disallow: *viewfindertypemv_facet*
376Disallow: *virtualrealitycompatibility_facet*
377Disallow: *wanportssv_facet*
378Disallow: *wearingstyle_facet*
379Disallow: *currentprice2_facet*
380Disallow: *customerreviews_facet*Customer*Rating*5*
381Disallow: *customerreviews_facet*Customer*Rating*4*
382Disallow: *customerreviews_facet*Customer*Rating*3*
383Disallow: *customerreviews_facet*Customer*Rating*2*
384Disallow: *customerreviews_facet*Customer*Rating*1*
385Disallow: *soldout_facet*
386Disallow: */site/promo/*?*_facet*
387Disallow: */product/list/*
388
389# Pagination Crawl Trap Controls
390Disallow: *cp=1*
391Disallow: *cp=20*
392Disallow: *cp=6*
393Disallow: *cp=7*
394Disallow: *cp=8*
395Disallow: *cp=9*
396Disallow: *?*page=*&page=*$
397
398# Search Parameters Controls
399Disallow: */site/searchpage.jsp?st=*
400Disallow: *?id*_facet*&st=*
401Disallow: *?id*_facet*_facet*&st=*
402Disallow: *?cp=*&id=*&st=*
403
404# Malformed URL Traps
405Disallow: *==$
406Disallow: */-?
407Disallow: */undefined$
408Disallow: */null$
409Disallow: *javascript:winopengld*
410
411# ==================================================
412# BUCKET 4: PAID SEARCH / ADS / MEDIA BOTS
413# Leaves pathways unobstructed for digital advertising synchronization.
414# ==================================================
415User-agent: AdsBot-Google
416User-agent: Googlebot-Image
417User-agent: Mediapartners-Google
418User-agent: Google-Display-Ads-Bot
419Disallow:
420
421# ==================================================
422# BUCKET 5: BLOCKED AGGRESSIVE THIRD-PARTY AGGREGATORS / TOOLS
423# Absolute block to preserve server performance and block unauthorized scrapers.
424# ==================================================
425User-agent: GetIntentCrawler
426User-agent: DotBot
427User-agent: MJ12bot
428User-agent: thesis-research-bot
429User-agent: TerraCotta
430User-agent: BigSurAI
431User-agent: MyCentralAIScraper
432Disallow: / [2.1]
433
434# ==================================================
435# DATA SITEMAPS
436# ==================================================
437Sitemap: https://sitemaps.bestbuy.com/sitemaps_discover_learn.xml
438Sitemap: https://sitemaps.bestbuy.com/sitemaps_pdp.xml
439Sitemap: https://sitemaps.bestbuy.com/sitemaps_promos.xml
440Sitemap: https://sitemaps.bestbuy.com/sitemaps_qna.xml
441Sitemap: https://sitemaps.bestbuy.com/sitemaps_rnr.xml
442Sitemap: https://sitemaps.bestbuy.com/sitemaps_search_plps.xml
443Sitemap: https://sitemaps.bestbuy.com/sitemaps_standalone_qa.xml
444Sitemap: https://sitemaps.bestbuy.com/sitemaps_browse_plps.xml
445Sitemap: https://sitemaps.bestbuy.com/sitemaps_manually_curated_pages.xml
446Sitemap: https://sitemaps.bestbuy.com/sitemaps_combos.xml
447Sitemap: https://sitemaps.bestbuy.com/sitemaps_bsin.xml