NeuralCrawl

Best Buy / robots.txt snapshot

← back to bestbuy.com · fetched 2026-11-05T07:45:14Z (2d ago) · HTTP 200 · 14823 bytes · sha256 a52ad62f31ed5b42 · raw

final URL: https://www.bestbuy.com/robots.txt

1# --------------------------------------------------
2# BEST BUY OFFICIAL ENTERPRISE ROBOTS.TXT
3# Optimized for search engines, AI discovery, ads, and crawl control
4# --------------------------------------------------
5
6# ==================================================
7# BUCKET 1: AI SEARCH & DISCOVERY BOT ACCESS
8# Allows conversational answer engines to securely access clean landing pages.
9# Mirrors explicit production parameters to protect long-tail revenue.
10# ==================================================
11User-agent: OAI-SearchBot
12User-agent: ChatGPT-User
13User-agent: perplexity-user
14User-agent: GPTBot
15User-agent: Claude-User
16User-agent: Claudebot
17User-agent: PerplexityBot
18User-agent: Applebot
19User-agent: Applebot-Extended
20User-agent: Google-Extended
21User-agent: Gemini-AI
22User-agent: Gemini-Deep-Research
23User-agent: GoogleAgent-Mariner
24User-agent: AmazonBot
25User-agent: GrokBot
26User-agent: xAI-Grok
27User-agent: Diffbot
28User-agent: Meta-ExternalAgent
29User-agent: Meta-ExternalFetcher
30User-agent: DuckAssistBot
31User-agent: Brightbot
32User-agent: webzio
33
34Allow: /
35Allow: /site/
36Allow: /discover-learn/
37
38# System Application Blockouts
39Disallow: /nex/
40Disallow: /m/e/*
41Disallow: /browse-api/*
42Disallow: /api/*.json
43Disallow: /availability/*
44Disallow: /cart
45Disallow: /cart/*
46Disallow: /cartitems/*
47Disallow: /fulfillment/*
48Disallow: /location/*
49Disallow: /pricing/*
50Disallow: /productfulfillment/*
51Disallow: /profile/*
52Disallow: /services/plans/*
53Disallow: /services/shoppingscheduler*
54Disallow: /site/builder/*
55Disallow: /site/modals/*
56Disallow: /site/dni/*
57Disallow: /site/offerbuilder/*
58
59# User Dynamic Input Paths
60Disallow: /site/questions/submission*
61Disallow: /site/answers/submission*
62Disallow: /site/reviews/submission*
63
64# Production-Safe Explicit Tab Fragments
65Disallow: /*template=_gameDetailsTab
66Disallow: /*template=_movieDetailsTab
67Disallow: /*template=_musicDetailsTab
68Disallow: /*template=_softwareDetailsTab
69Disallow: /*template=_accessoriesTab
70Disallow: /*template=_castAndCrewTab
71Disallow: /*template=_editorialTab
72Disallow: /*template=_episodesTab
73Disallow: /*template=_protectionAndServicesTab
74Disallow: /*template=_specificationsTab
75Disallow: /*template=_featuresSpoke
76Disallow: /*template=_buyingOptionsNewTab
77
78# Production-Safe Explicit Core Facet Controls (Brand/Category left open)
79Disallow: *armodelformat_facet*
80Disallow: *awardtype_facet*
81Disallow: *awardwinner_facet*
82Disallow: *awardyear_facet*
83Disallow: *blenderfunctions_facet*
84Disallow: *boxdesign_facet*
85Disallow: *cast_facet*
86Disallow: *cleaningpathwidthrange_facet*
87Disallow: *consumerlifestage_facet*
88Disallow: *decade_facet*
89Disallow: *defrosttype_facet*
90Disallow: *departmentid_facet*
91Disallow: *depthtype_facet*
92Disallow: *displayport_facet*
93Disallow: *doorstylesv_facet*
94Disallow: *drawertype_facet*
95Disallow: *dynamiccategory_facet*
96Disallow: *effectivemegapixels_facet*
97Disallow: *flashmodesmv_facet*
98Disallow: *framematerialmv_facet*
99Disallow: *giftcardthemesv_facet*
100Disallow: *handletype_facet*
101Disallow: *headsetstylesv_facet*
102Disallow: *heatingsource_facet*
103Disallow: *imagesensorsize_facet*
104Disallow: *imagesensortype11_facet*
105Disallow: *includedcolors_facet*
106Disallow: *inkmodelcompatibilitymv_facet*
107Disallow: *internalorexternal_facet*
108Disallow: *laptopuse_facet*
109Disallow: *lengthofcable_facet*
110Disallow: *mattressmaterial_facet*
111Disallow: *maximumsensitivity_facet*
112Disallow: *minfocallengthrange_facet*
113Disallow: *mobileinternalmemory_facet*
114Disallow: *monitorresponsetimerange_facet*
115Disallow: *monitorsnumberofusb30typecports_facet*
116Disallow: *monitorsnumberofusb31typecports_facet*
117Disallow: *mountmaterial_facet*
118Disallow: *networkingcabletype_facet*
119Disallow: *numberofbedroomsef_facet*
120Disallow: *numberofburnerssv_facet*
121Disallow: *numberofbuttons_facet*
122Disallow: *numberofdryingcycles_facet*
123Disallow: *numberoffanspeedssv_facet*
124Disallow: *numberofhdmiinputssv_facet*
125Disallow: *numberoflanports_facet*
126Disallow: *numberofspeakers_facet*
127Disallow: *numberofspeedssv_facet*
128Disallow: *numberofthunderbolt_ports_facet*
129Disallow: *numberofwashrinsecycle_facet*
130Disallow: *papersize_facet*
131Disallow: *playbackformatsmv_facet*
132Disallow: *productcategories_facet*
133Disallow: *screensizediagonalrange_facet*
134Disallow: *securityconnectivity_facet*
135Disallow: *securityresolution_facet*
136Disallow: *securitystoragetype_facet*
137Disallow: *softwareformat_facet*
138Disallow: *sqbprocessormodel_facet*
139Disallow: *sqbtotalstoragecapacity_facet*
140Disallow: *stringtype_facet*
141Disallow: *subclassid_facet*
142Disallow: *subgenre_facet*
143Disallow: *thruthedoordispenser_facet*
144Disallow: *toastingslotssv_facet*
145Disallow: *totaltechdeals_facet*
146Disallow: *touchscreentype_facet*
147Disallow: *transferrate_facet*
148Disallow: *transistorsize_facet*
149Disallow: *transmitterdistance_facet*
150Disallow: *treeheight_facet*
151Disallow: *treelightbulbcolor_facet*
152Disallow: *tubfinishmv_facet*
153Disallow: *varyinganglescreen_facet*
154Disallow: *vendorcollection_facet*
155Disallow: *videomemory_facet*
156Disallow: *viewfindertypemv_facet*
157Disallow: *virtualrealitycompatibility_facet*
158Disallow: *wanportssv_facet*
159Disallow: *wearingstyle_facet*
160Disallow: *currentprice2_facet*
161Disallow: *customerreviews_facet*Customer*Rating*5*
162Disallow: *customerreviews_facet*Customer*Rating*4*
163Disallow: *customerreviews_facet*Customer*Rating*3*
164Disallow: *customerreviews_facet*Customer*Rating*2*
165Disallow: *customerreviews_facet*Customer*Rating*1*
166Disallow: *soldout_facet*
167Disallow: */site/promo/*?*_facet*
168Disallow: */product/list/*
169Disallow: */awacs-ingestor/*
170
171# Exact Pagination Safeguards
172Disallow: *cp=1*
173Disallow: *cp=20*
174Disallow: *cp=6*
175Disallow: *cp=7*
176Disallow: *cp=8*
177Disallow: *cp=9*
178Disallow: *?*page=*&page=*$
179
180# Malformed URL Traps (Universal Protection)
181Disallow: *==$
182Disallow: */-?
183Disallow: */undefined$
184Disallow: */null$
185Disallow: *javascript:winopengld*
186
187# ==================================================
188# BUCKET 2: AI TRAINING / DATA INGESTION FIREWALL
189# Completely blocks models from scraping copy assets for foundational training.
190# ==================================================
191User-agent: GPTBot-Web
192User-agent: CCBot
193User-agent: Anthropic-AI
194User-agent: Claude-Web
195User-agent: Bytespider
196
197# ==================================================
198# BUCKET 3: UNIFIED WEB ENGINES & UNIVERSAL DEFAULTS
199# Consolidated grouping ensures zero functional divergence between indexing bots.
200# ==================================================
201User-agent: Googlebot
202User-agent: Bingbot
203User-agent: DuckDuckBot
204User-agent: Slurp
205User-agent: *
206
207# Standard Asset Pathways
208Allow: /~assets/
209Allow: /discover-learn/
210
211# Core System Blockouts
212Disallow: /nex/
213Disallow: /m/e/*
214Disallow: /site/builder/*
215Disallow: /browse-api/*
216Disallow: /api/*.json
217Disallow: /availability/*
218Disallow: /cart
219Disallow: /cart/*
220Disallow: /cartitems/*
221Disallow: /fulfillment/*
222Disallow: /location/*
223Disallow: /pricing/*
224Disallow: /productfulfillment/*
225Disallow: /profile/*
226Disallow: /site/modals/*
227Disallow: /services/plans/*
228Disallow: /site/dni/*
229Disallow: /site/offerbuilder/*
230Disallow: /services/shoppingscheduler*
231
232# Staging, Internal Test, & Promotional Exclusion Matrices
233Disallow: /site/clp/test-page/*
234Disallow: /site/test-page/*
235Disallow: /site/misc/test-pages/*
236Disallow: /site/offer/*
237Disallow: /site/compare?skus=*
238Disallow: /site/clp/conditional-offers/*
239Disallow: /site/total-tech-support/*
240Disallow: /site/clp/annette-test-listing/*
241Disallow: /site/shop-view-test-page/*
242Disallow: /site/clp/test/*
243Disallow: /site/test-cloud-ready/*
244Disallow: /site/clp/mystery-rewards-certificate/pcmcat1524134875052.c?id=pcmcat1524134875052
245Disallow: /purchasehistory/gift-receipt*
246Disallow: /contact-us?*
247Disallow: /ease/email
248Disallow: /ihc/invoice*
249Disallow: /identity/newAccount?*
250Disallow: /identity/signin?*
251Disallow: /identity/signin/forgotPassword?*
252Disallow: /dossier/service/appointment/*/*-*
253Disallow: /dossier/service/appointment/search?*
254Disallow: */product-selector/*
255Disallow: */email-subscription/*
256
257# Legacy Template Controller Protections (Parameter Strictness Maintained)
258Disallow: /site/olspage.jsp?id=pcat17098*
259Disallow: /site/olstemplatemapper.jsp?id=pcat17098*
260Disallow: /site/olspage.jsp?id=pcat17089*
261Disallow: /site/olstemplatemapper.jsp?id=pcat17089*
262Disallow: /site/olspage.jsp?id=cat13503*
263Disallow: /site/olstemplatemapper.jsp?id=cat13503*
264Disallow: /site/olspage.jsp?id=pcat17007*
265Disallow: /site/olstemplatemapper.jsp?id=pcat17007*
266Disallow: /site/olspage.jsp?id=pcat17408*
267Disallow: /site/olstemplatemapper.jsp?id=pcat17408*
268Disallow: /site/olspage.jsp?id=cat12091*
269Disallow: /site/olstemplatemapper.jsp?id=cat12091*
270Disallow: /site/olspage.jsp?id=pcat17005*
271Disallow: /site/olstemplatemapper.jsp?id=pcat17005*
272Disallow: /site/olspage.jsp?id=cat12076*
273Disallow: /site/olstemplatemapper.jsp?id=cat12076*
274Disallow: /site/olspage.jsp?id=cat12080*
275Disallow: /site/olstemplatemapper.jsp?id=cat12080*
276Disallow: /site/olspage.jsp?id=cat12101*
277Disallow: /site/olstemplatemapper.jsp?id=cat12101*
278Disallow: /site/olstemplatemapper.jsp?id=pcat17006*
279Disallow: /site/olspage.jsp?id=pcat17006*
280Disallow: */site/olspage.jsp*
281
282# Asynchronous Layout Fragment Controls
283Disallow: /*template=_gameDetailsTab
284Disallow: /*template=_movieDetailsTab
285Disallow: /*template=_musicDetailsTab
286Disallow: /*template=_softwareDetailsTab
287Disallow: /*template=_accessoriesTab
288Disallow: /*template=_castAndCrewTab
289Disallow: /*template=_editorialTab
290Disallow: /*template=_episodesTab
291Disallow: /*template=_protectionAndServicesTab
292Disallow: /*template=_specificationsTab
293Disallow: /*template=_featuresSpoke
294Disallow: /*template=_buyingOptionsNewTab
295
296# User Input Paths
297Disallow: /site/questions/submission*
298Disallow: /site/answers/submission*
299Disallow: /site/reviews/submission*
300
301# Granular Facet Exclusions
302Disallow: *armodelformat_facet*
303Disallow: *awardtype_facet*
304Disallow: *awardwinner_facet*
305Disallow: *awardyear_facet*
306Disallow: *blenderfunctions_facet*
307Disallow: *boxdesign_facet*
308Disallow: *cast_facet*
309Disallow: *cleaningpathwidthrange_facet*
310Disallow: *consumerlifestage_facet*
311Disallow: *decade_facet*
312Disallow: *defrosttype_facet*
313Disallow: *departmentid_facet*
314Disallow: *depthtype_facet*
315Disallow: *displayport_facet*
316Disallow: *doorstylesv_facet*
317Disallow: *drawertype_facet*
318Disallow: *dynamiccategory_facet*
319Disallow: *effectivemegapixels_facet*
320Disallow: *flashmodesmv_facet*
321Disallow: *framematerialmv_facet*
322Disallow: *giftcardthemesv_facet*
323Disallow: *handletype_facet*
324Disallow: *headsetstylesv_facet*
325Disallow: *heatingsource_facet*
326Disallow: *imagesensorsize_facet*
327Disallow: *imagesensortype11_facet*
328Disallow: *includedcolors_facet*
329Disallow: *inkmodelcompatibilitymv_facet*
330Disallow: *internalorexternal_facet*
331Disallow: *laptopuse_facet*
332Disallow: *lengthofcable_facet*
333Disallow: *mattressmaterial_facet*
334Disallow: *maximumsensitivity_facet*
335Disallow: *minfocallengthrange_facet*
336Disallow: *mobileinternalmemory_facet*
337Disallow: *monitorresponsetimerange_facet*
338Disallow: *monitorsnumberofusb30typecports_facet*
339Disallow: *monitorsnumberofusb31typecports_facet*
340Disallow: *mountmaterial_facet*
341Disallow: *networkingcabletype_facet*
342Disallow: *numberofbedroomsef_facet*
343Disallow: *numberofburnerssv_facet*
344Disallow: *numberofbuttons_facet*
345Disallow: *numberofdryingcycles_facet*
346Disallow: *numberoffanspeedssv_facet*
347Disallow: *numberofhdmiinputssv_facet*
348Disallow: *numberoflanports_facet*
349Disallow: *numberofspeakers_facet*
350Disallow: *numberofspeedssv_facet*
351Disallow: *numberofthunderbolt_ports_facet*
352Disallow: *numberofwashrinsecycle_facet*
353Disallow: *papersize_facet*
354Disallow: *playbackformatsmv_facet*
355Disallow: *productcategories_facet*
356Disallow: *screensizediagonalrange_facet*
357Disallow: *securityconnectivity_facet*
358Disallow: *securityresolution_facet*
359Disallow: *securitystoragetype_facet*
360Disallow: *softwareformat_facet*
361Disallow: *sqbprocessormodel_facet*
362Disallow: *sqbtotalstoragecapacity_facet*
363Disallow: *stringtype_facet*
364Disallow: *subclassid_facet*
365Disallow: *subgenre_facet*
366Disallow: *thruthedoordispenser_facet*
367Disallow: *toastingslotssv_facet*
368Disallow: *totaltechdeals_facet*
369Disallow: *touchscreentype_facet*
370Disallow: *transferrate_facet*
371Disallow: *transistorsize_facet*
372Disallow: *transmitterdistance_facet*
373Disallow: *treeheight_facet*
374Disallow: *treelightbulbcolor_facet*
375Disallow: *tubfinishmv_facet*
376Disallow: *varyinganglescreen_facet*
377Disallow: *vendorcollection_facet*
378Disallow: *videomemory_facet*
379Disallow: *viewfindertypemv_facet*
380Disallow: *virtualrealitycompatibility_facet*
381Disallow: *wanportssv_facet*
382Disallow: *wearingstyle_facet*
383Disallow: *currentprice2_facet*
384Disallow: *customerreviews_facet*Customer*Rating*5*
385Disallow: *customerreviews_facet*Customer*Rating*4*
386Disallow: *customerreviews_facet*Customer*Rating*3*
387Disallow: *customerreviews_facet*Customer*Rating*2*
388Disallow: *customerreviews_facet*Customer*Rating*1*
389Disallow: *soldout_facet*
390Disallow: */site/promo/*?*_facet*
391Disallow: */product/list/*
392
393# Pagination Crawl Trap Controls
394Disallow: *cp=1*
395Disallow: *cp=20*
396Disallow: *cp=6*
397Disallow: *cp=7*
398Disallow: *cp=8*
399Disallow: *cp=9*
400Disallow: *?*page=*&page=*$
401
402# Malformed URL Traps
403Disallow: *==$
404Disallow: */-?
405Disallow: */undefined$
406Disallow: */null$
407Disallow: *javascript:winopengld*
408
409# ==================================================
410# BUCKET 4: PAID SEARCH / ADS / MEDIA BOTS
411# Leaves pathways unobstructed for digital advertising synchronization.
412# ==================================================
413User-agent: AdsBot-Google
414User-agent: Googlebot-Image
415User-agent: Mediapartners-Google
416User-agent: Google-Display-Ads-Bot
417Disallow:
418
419# ==================================================
420# BUCKET 5: BLOCKED AGGRESSIVE THIRD-PARTY AGGREGATORS / TOOLS
421# Absolute block to preserve server performance and block unauthorized scrapers.
422# ==================================================
423User-agent: GetIntentCrawler
424User-agent: DotBot
425User-agent: MJ12bot
426User-agent: thesis-research-bot
427User-agent: TerraCotta
428User-agent: BigSurAI
429User-agent: MyCentralAIScraper
430Disallow: /
431
432# ==================================================
433# DATA SITEMAPS
434# ==================================================
435Sitemap: https://sitemaps.bestbuy.com/sitemaps_discover_learn.xml
436Sitemap: https://sitemaps.bestbuy.com/sitemaps_pdp.xml
437Sitemap: https://sitemaps.bestbuy.com/sitemaps_promos.xml
438Sitemap: https://sitemaps.bestbuy.com/sitemaps_qna.xml
439Sitemap: https://sitemaps.bestbuy.com/sitemaps_rnr.xml
440Sitemap: https://sitemaps.bestbuy.com/sitemaps_search_plps.xml
441Sitemap: https://sitemaps.bestbuy.com/sitemaps_standalone_qa.xml
442Sitemap: https://sitemaps.bestbuy.com/sitemaps_browse_plps.xml
443Sitemap: https://sitemaps.bestbuy.com/sitemaps_manually_curated_pages.xml
444Sitemap: https://sitemaps.bestbuy.com/sitemaps_combos.xml
445Sitemap: https://sitemaps.bestbuy.com/sitemaps_bsin.xml