NeuralCrawl

Best Buy / robots.txt snapshot

← back to bestbuy.com · fetched 2026-09-17T15:11:14Z (1d ago) · HTTP 200 · 14583 bytes · sha256 9267b879d0bbc100 · raw

final URL: https://www.bestbuy.com/robots.txt

1# --------------------------------------------------
2# BEST BUY OFFICIAL ENTERPRISE ROBOTS.TXT
3# Optimized for search engines, AI discovery, ads, and crawl control
4# --------------------------------------------------
5
6# ==================================================
7# BUCKET 1: AI SEARCH & DISCOVERY BOT ACCESS
8# Allows conversational answer engines to securely access clean landing pages.
9# Mirrors explicit production parameters to protect long-tail revenue.
10# ==================================================
11User-agent: OAI-SearchBot
12User-agent: ChatGPT-User
13User-agent: Perplexity-User
14User-agent: GPTBot
15User-agent: Claude-User
16User-agent: Claudebot
17User-agent: PerplexityBot
18User-agent: Applebot
19User-agent: Google-Extended
20User-agent: Gemeni-AI
21User-agent: Gemini-Deep-Research
22User-agent: AmazonBot
23
24Allow: /site/
25Allow: /discover-learn/
26
27# System Application Blockouts
28Disallow: /nex/
29Disallow: /m/e/*
30Disallow: /browse-api/*
31Disallow: /api/*.json
32Disallow: /availability/*
33Disallow: /cart
34Disallow: /cart/*
35Disallow: /cartitems/*
36Disallow: /fulfillment/*
37Disallow: /location/*
38Disallow: /pricing/*
39Disallow: /productfulfillment/*
40Disallow: /profile/*
41Disallow: /services/plans/*
42Disallow: /services/shoppingscheduler*
43Disallow: /site/builder/*
44Disallow: /site/modals/*
45Disallow: /site/dni/*
46Disallow: /site/offerbuilder/*
47
48# User Dynamic Input Paths
49Disallow: /site/questions/submission*
50Disallow: /site/answers/submission*
51Disallow: /site/reviews/submission*
52
53# Production-Safe Explicit Tab Fragments
54Disallow: /*template=_gameDetailsTab
55Disallow: /*template=_movieDetailsTab
56Disallow: /*template=_musicDetailsTab
57Disallow: /*template=_softwareDetailsTab
58Disallow: /*template=_accessoriesTab
59Disallow: /*template=_castAndCrewTab
60Disallow: /*template=_editorialTab
61Disallow: /*template=_episodesTab
62Disallow: /*template=_protectionAndServicesTab
63Disallow: /*template=_specificationsTab
64Disallow: /*template=_featuresSpoke
65Disallow: /*template=_buyingOptionsNewTab
66
67# Production-Safe Explicit Core Facet Controls (Brand/Category left open)
68Disallow: *armodelformat_facet*
69Disallow: *awardtype_facet*
70Disallow: *awardwinner_facet*
71Disallow: *awardyear_facet*
72Disallow: *blenderfunctions_facet*
73Disallow: *boxdesign_facet*
74Disallow: *cast_facet*
75Disallow: *cleaningpathwidthrange_facet*
76Disallow: *consumerlifestage_facet*
77Disallow: *decade_facet*
78Disallow: *defrosttype_facet*
79Disallow: *departmentid_facet*
80Disallow: *depthtype_facet*
81Disallow: *displayport_facet*
82Disallow: *doorstylesv_facet*
83Disallow: *drawertype_facet*
84Disallow: *dynamiccategory_facet*
85Disallow: *effectivemegapixels_facet*
86Disallow: *flashmodesmv_facet*
87Disallow: *framematerialmv_facet*
88Disallow: *giftcardthemesv_facet*
89Disallow: *handletype_facet*
90Disallow: *headsetstylesv_facet*
91Disallow: *heatingsource_facet*
92Disallow: *imagesensorsize_facet*
93Disallow: *imagesensortype11_facet*
94Disallow: *includedcolors_facet*
95Disallow: *inkmodelcompatibilitymv_facet*
96Disallow: *internalorexternal_facet*
97Disallow: *laptopuse_facet*
98Disallow: *lengthofcable_facet*
99Disallow: *mattressmaterial_facet*
100Disallow: *maximumsensitivity_facet*
101Disallow: *minfocallengthrange_facet*
102Disallow: *mobileinternalmemory_facet*
103Disallow: *monitorresponsetimerange_facet*
104Disallow: *monitorsnumberofusb30typecports_facet*
105Disallow: *monitorsnumberofusb31typecports_facet*
106Disallow: *mountmaterial_facet*
107Disallow: *networkingcabletype_facet*
108Disallow: *numberofbedroomsef_facet*
109Disallow: *numberofburnerssv_facet*
110Disallow: *numberofbuttons_facet*
111Disallow: *numberofdryingcycles_facet*
112Disallow: *numberoffanspeedssv_facet*
113Disallow: *numberofhdmiinputssv_facet*
114Disallow: *numberoflanports_facet*
115Disallow: *numberofspeakers_facet*
116Disallow: *numberofspeedssv_facet*
117Disallow: *numberofthunderbolt_ports_facet*
118Disallow: *numberofwashrinsecycle_facet*
119Disallow: *papersize_facet*
120Disallow: *playbackformatsmv_facet*
121Disallow: *productcategories_facet*
122Disallow: *screensizediagonalrange_facet*
123Disallow: *securityconnectivity_facet*
124Disallow: *securityresolution_facet*
125Disallow: *securitystoragetype_facet*
126Disallow: *softwareformat_facet*
127Disallow: *sqbprocessormodel_facet*
128Disallow: *sqbtotalstoragecapacity_facet*
129Disallow: *stringtype_facet*
130Disallow: *subclassid_facet*
131Disallow: *subgenre_facet*
132Disallow: *thruthedoordispenser_facet*
133Disallow: *toastingslotssv_facet*
134Disallow: *totaltechdeals_facet*
135Disallow: *touchscreentype_facet*
136Disallow: *transferrate_facet*
137Disallow: *transistorsize_facet*
138Disallow: *transmitterdistance_facet*
139Disallow: *treeheight_facet*
140Disallow: *treelightbulbcolor_facet*
141Disallow: *tubfinishmv_facet*
142Disallow: *varyinganglescreen_facet*
143Disallow: *vendorcollection_facet*
144Disallow: *videomemory_facet*
145Disallow: *viewfindertypemv_facet*
146Disallow: *virtualrealitycompatibility_facet*
147Disallow: *wanportssv_facet*
148Disallow: *wearingstyle_facet*
149Disallow: *currentprice2_facet*
150Disallow: *customerreviews_facet*Customer*Rating*5*
151Disallow: *customerreviews_facet*Customer*Rating*4*
152Disallow: *customerreviews_facet*Customer*Rating*3*
153Disallow: *customerreviews_facet*Customer*Rating*2*
154Disallow: *customerreviews_facet*Customer*Rating*1*
155Disallow: *soldout_facet*
156Disallow: */site/promo/*?*_facet*
157Disallow: */product/list/*
158Disallow: */awacs-ingestor/*
159
160# Exact Pagination Safeguards
161Disallow: *cp=1*
162Disallow: *cp=20*
163Disallow: *cp=6*
164Disallow: *cp=7*
165Disallow: *cp=8*
166Disallow: *cp=9*
167Disallow: *?*page=*&page=*$
168
169# Malformed URL Traps (Universal Protection)
170Disallow: *==$
171Disallow: */-?
172Disallow: */undefined$
173Disallow: */null$
174Disallow: *javascript:winopengld*
175
176# ==================================================
177# BUCKET 2: AI TRAINING / DATA INGESTION FIREWALL
178# Completely blocks models from scraping copy assets for foundational training.
179# ==================================================
180User-agent: GPTBot-Web
181User-agent: CCBot
182User-agent: Anthropic-AI
183User-agent: Claude-Web
184User-agent: Bytespider
185Disallow: /[1.1]
186
187# ==================================================
188# BUCKET 3: UNIFIED WEB ENGINES & UNIVERSAL DEFAULTS
189# Consolidated grouping ensures zero functional divergence between indexing bots.
190# ==================================================
191User-agent: Googlebot
192User-agent: Bingbot
193User-agent: DuckDuckBot
194User-agent: Slurp
195User-agent: *
196
197# Standard Asset Pathways
198Allow: /~assets/
199Allow: /discover-learn/
200
201# Core System Blockouts
202Disallow: /nex/
203Disallow: /m/e/*
204Disallow: /site/builder/*
205Disallow: /browse-api/*
206Disallow: /api/*.json
207Disallow: /availability/*
208Disallow: /cart
209Disallow: /cart/*
210Disallow: /cartitems/*
211Disallow: /fulfillment/*
212Disallow: /location/*
213Disallow: /pricing/*
214Disallow: /productfulfillment/*
215Disallow: /profile/*
216Disallow: /site/modals/*
217Disallow: /services/plans/*
218Disallow: /site/dni/*
219Disallow: /site/offerbuilder/*
220Disallow: /services/shoppingscheduler*
221
222# Staging, Internal Test, & Promotional Exclusion Matrices
223Disallow: /site/clp/test-page/*
224Disallow: /site/test-page/*
225Disallow: /site/misc/test-pages/*
226Disallow: /site/offer/*
227Disallow: /site/compare?skus=*
228Disallow: /site/clp/conditional-offers/*
229Disallow: /site/total-tech-support/*
230Disallow: /site/clp/annette-test-listing/*
231Disallow: /site/shop-view-test-page/*
232Disallow: /site/clp/test/*
233Disallow: /site/test-cloud-ready/*
234Disallow: /site/clp/mystery-rewards-certificate/pcmcat1524134875052.c?id=pcmcat1524134875052
235Disallow: /purchasehistory/gift-receipt*
236Disallow: /contact-us?*
237Disallow: /ease/email
238Disallow: /ihc/invoice*
239Disallow: /identity/newAccount?*
240Disallow: /identity/signin?*
241Disallow: /identity/signin/forgotPassword?*
242Disallow: /dossier/service/appointment/*/*-*
243Disallow: /dossier/service/appointment/search?*
244Disallow: */product-selector/*
245Disallow: */email-subscription/*
246
247# Legacy Template Controller Protections (Parameter Strictness Maintained)
248Disallow: /site/olspage.jsp?id=pcat17098*
249Disallow: /site/olstemplatemapper.jsp?id=pcat17098*
250Disallow: /site/olspage.jsp?id=pcat17089*
251Disallow: /site/olstemplatemapper.jsp?id=pcat17089*
252Disallow: /site/olspage.jsp?id=cat13503*
253Disallow: /site/olstemplatemapper.jsp?id=cat13503*
254Disallow: /site/olspage.jsp?id=pcat17007*
255Disallow: /site/olstemplatemapper.jsp?id=pcat17007*
256Disallow: /site/olspage.jsp?id=pcat17408*
257Disallow: /site/olstemplatemapper.jsp?id=pcat17408*
258Disallow: /site/olspage.jsp?id=cat12091*
259Disallow: /site/olstemplatemapper.jsp?id=cat12091*
260Disallow: /site/olspage.jsp?id=pcat17005*
261Disallow: /site/olstemplatemapper.jsp?id=pcat17005*
262Disallow: /site/olspage.jsp?id=cat12076*
263Disallow: /site/olstemplatemapper.jsp?id=cat12076*
264Disallow: /site/olspage.jsp?id=cat12080*
265Disallow: /site/olstemplatemapper.jsp?id=cat12080*
266Disallow: /site/olspage.jsp?id=cat12101*
267Disallow: /site/olstemplatemapper.jsp?id=cat12101*
268Disallow: /site/olstemplatemapper.jsp?id=pcat17006*
269Disallow: /site/olspage.jsp?id=pcat17006*
270Disallow: */site/olspage.jsp*
271
272# Asynchronous Layout Fragment Controls
273Disallow: /*template=_gameDetailsTab
274Disallow: /*template=_movieDetailsTab
275Disallow: /*template=_musicDetailsTab
276Disallow: /*template=_softwareDetailsTab
277Disallow: /*template=_accessoriesTab
278Disallow: /*template=_castAndCrewTab
279Disallow: /*template=_editorialTab
280Disallow: /*template=_episodesTab
281Disallow: /*template=_protectionAndServicesTab
282Disallow: /*template=_specificationsTab
283Disallow: /*template=_featuresSpoke
284Disallow: /*template=_buyingOptionsNewTab
285
286# User Input Paths
287Disallow: /site/questions/submission*
288Disallow: /site/answers/submission*
289Disallow: /site/reviews/submission*
290
291# Granular Facet Exclusions
292Disallow: *armodelformat_facet*
293Disallow: *awardtype_facet*
294Disallow: *awardwinner_facet*
295Disallow: *awardyear_facet*
296Disallow: *blenderfunctions_facet*
297Disallow: *boxdesign_facet*
298Disallow: *cast_facet*
299Disallow: *cleaningpathwidthrange_facet*
300Disallow: *consumerlifestage_facet*
301Disallow: *decade_facet*
302Disallow: *defrosttype_facet*
303Disallow: *departmentid_facet*
304Disallow: *depthtype_facet*
305Disallow: *displayport_facet*
306Disallow: *doorstylesv_facet*
307Disallow: *drawertype_facet*
308Disallow: *dynamiccategory_facet*
309Disallow: *effectivemegapixels_facet*
310Disallow: *flashmodesmv_facet*
311Disallow: *framematerialmv_facet*
312Disallow: *giftcardthemesv_facet*
313Disallow: *handletype_facet*
314Disallow: *headsetstylesv_facet*
315Disallow: *heatingsource_facet*
316Disallow: *imagesensorsize_facet*
317Disallow: *imagesensortype11_facet*
318Disallow: *includedcolors_facet*
319Disallow: *inkmodelcompatibilitymv_facet*
320Disallow: *internalorexternal_facet*
321Disallow: *laptopuse_facet*
322Disallow: *lengthofcable_facet*
323Disallow: *mattressmaterial_facet*
324Disallow: *maximumsensitivity_facet*
325Disallow: *minfocallengthrange_facet*
326Disallow: *mobileinternalmemory_facet*
327Disallow: *monitorresponsetimerange_facet*
328Disallow: *monitorsnumberofusb30typecports_facet*
329Disallow: *monitorsnumberofusb31typecports_facet*
330Disallow: *mountmaterial_facet*
331Disallow: *networkingcabletype_facet*
332Disallow: *numberofbedroomsef_facet*
333Disallow: *numberofburnerssv_facet*
334Disallow: *numberofbuttons_facet*
335Disallow: *numberofdryingcycles_facet*
336Disallow: *numberoffanspeedssv_facet*
337Disallow: *numberofhdmiinputssv_facet*
338Disallow: *numberoflanports_facet*
339Disallow: *numberofspeakers_facet*
340Disallow: *numberofspeedssv_facet*
341Disallow: *numberofthunderbolt_ports_facet*
342Disallow: *numberofwashrinsecycle_facet*
343Disallow: *papersize_facet*
344Disallow: *playbackformatsmv_facet*
345Disallow: *productcategories_facet*
346Disallow: *screensizediagonalrange_facet*
347Disallow: *securityconnectivity_facet*
348Disallow: *securityresolution_facet*
349Disallow: *securitystoragetype_facet*
350Disallow: *softwareformat_facet*
351Disallow: *sqbprocessormodel_facet*
352Disallow: *sqbtotalstoragecapacity_facet*
353Disallow: *stringtype_facet*
354Disallow: *subclassid_facet*
355Disallow: *subgenre_facet*
356Disallow: *thruthedoordispenser_facet*
357Disallow: *toastingslotssv_facet*
358Disallow: *totaltechdeals_facet*
359Disallow: *touchscreentype_facet*
360Disallow: *transferrate_facet*
361Disallow: *transistorsize_facet*
362Disallow: *transmitterdistance_facet*
363Disallow: *treeheight_facet*
364Disallow: *treelightbulbcolor_facet*
365Disallow: *tubfinishmv_facet*
366Disallow: *varyinganglescreen_facet*
367Disallow: *vendorcollection_facet*
368Disallow: *videomemory_facet*
369Disallow: *viewfindertypemv_facet*
370Disallow: *virtualrealitycompatibility_facet*
371Disallow: *wanportssv_facet*
372Disallow: *wearingstyle_facet*
373Disallow: *currentprice2_facet*
374Disallow: *customerreviews_facet*Customer*Rating*5*
375Disallow: *customerreviews_facet*Customer*Rating*4*
376Disallow: *customerreviews_facet*Customer*Rating*3*
377Disallow: *customerreviews_facet*Customer*Rating*2*
378Disallow: *customerreviews_facet*Customer*Rating*1*
379Disallow: *soldout_facet*
380Disallow: */site/promo/*?*_facet*
381Disallow: */product/list/*
382
383# Pagination Crawl Trap Controls
384Disallow: *cp=1*
385Disallow: *cp=20*
386Disallow: *cp=6*
387Disallow: *cp=7*
388Disallow: *cp=8*
389Disallow: *cp=9*
390Disallow: *?*page=*&page=*$
391
392# Malformed URL Traps
393Disallow: *==$
394Disallow: */-?
395Disallow: */undefined$
396Disallow: */null$
397Disallow: *javascript:winopengld*
398
399# ==================================================
400# BUCKET 4: PAID SEARCH / ADS / MEDIA BOTS
401# Leaves pathways unobstructed for digital advertising synchronization.
402# ==================================================
403User-agent: AdsBot-Google
404User-agent: Googlebot-Image
405User-agent: Mediapartners-Google
406User-agent: Google-Display-Ads-Bot
407Disallow:
408
409# ==================================================
410# BUCKET 5: BLOCKED AGGRESSIVE THIRD-PARTY AGGREGATORS / TOOLS
411# Absolute block to preserve server performance and block unauthorized scrapers.
412# ==================================================
413User-agent: GetIntentCrawler
414User-agent: DotBot
415User-agent: MJ12bot
416User-agent: thesis-research-bot
417User-agent: TerraCotta
418User-agent: BigSurAI
419User-agent: MyCentralAIScraper
420Disallow: / [2.1]
421
422# ==================================================
423# DATA SITEMAPS
424# ==================================================
425Sitemap: https://sitemaps.bestbuy.com/sitemaps_discover_learn.xml
426Sitemap: https://sitemaps.bestbuy.com/sitemaps_pdp.xml
427Sitemap: https://sitemaps.bestbuy.com/sitemaps_promos.xml
428Sitemap: https://sitemaps.bestbuy.com/sitemaps_qna.xml
429Sitemap: https://sitemaps.bestbuy.com/sitemaps_rnr.xml
430Sitemap: https://sitemaps.bestbuy.com/sitemaps_search_plps.xml
431Sitemap: https://sitemaps.bestbuy.com/sitemaps_standalone_qa.xml
432Sitemap: https://sitemaps.bestbuy.com/sitemaps_browse_plps.xml
433Sitemap: https://sitemaps.bestbuy.com/sitemaps_manually_curated_pages.xml
434Sitemap: https://sitemaps.bestbuy.com/sitemaps_combos.xml
435Sitemap: https://sitemaps.bestbuy.com/sitemaps_bsin.xml