NeuralCrawl

Best Buy / robots.txt

← back to bestbuy.com · change detected 25d ago (2026-08-24T19:21:08Z) · modified +306 −76

What changed

Diff old (2026-06-20T01:10:30Z) → new (2026-08-24T19:21:08Z)

@@ -1,36 +1,33 @@
1- User-agent: *
2- User-agent: Googlebot
1+ # --------------------------------------------------
2+ # BEST BUY OFFICIAL ENTERPRISE ROBOTS.TXT
3+ # Optimized for search engines, AI discovery, ads, and crawl control
4+ # --------------------------------------------------
5+
6+ # ==================================================
7+ # BUCKET 1: AI SEARCH & DISCOVERY BOT ACCESS
8+ # Allows conversational answer engines to securely access clean landing pages.
9+ # Mirrors explicit production parameters to protect long-tail revenue.
10+ # ==================================================
11+ User-agent: OAI-SearchBot
12+ User-agent: ChatGPT-User
13+ User-agent: Perplexity-User
14+ User-agent: GPTBot
15+ User-agent: Claude-User
16+ User-agent: Claudebot
17+ User-agent: PerplexityBot
18+ User-agent: Applebot
19+ User-agent: Google-Extended
20+ User-agent: Gemeni-AI
21+ User-agent: Gemini-Deep-Research
22+ User-agent: AmazonBot
23+
24+ Allow: /site/
25+ Allow: /discover-learn/
26+
27+ # System Application Blockouts
328  Disallow: /nex/
429  Disallow: /m/e/*
5- Disallow: /site/builder/*
6- Disallow: /site/olspage.jsp?id=pcat17098*
7- Disallow: /site/olstemplatemapper.jsp?id=pcat17098*
8- Disallow: /site/olspage.jsp?id=pcat17089*
9- Disallow: /site/olstemplatemapper.jsp?id=pcat17089*
10- Disallow: /site/olspage.jsp?id=cat13503*
11- Disallow: /site/olstemplatemapper.jsp?id=cat13503*
12- Disallow: /site/olspage.jsp?id=pcat17007*
13- Disallow: /site/olstemplatemapper.jsp?id=pcat17007*
14- Disallow: /site/olspage.jsp?id=pcat17408*
15- Disallow: /site/olstemplatemapper.jsp?id=pcat17408*
16- Disallow: /site/olspage.jsp?id=cat12091*
17- Disallow: /site/olstemplatemapper.jsp?id=cat12091*
18- Disallow: /site/olspage.jsp?id=pcat17005*
19- Disallow: /site/olstemplatemapper.jsp?id=pcat17005*
20- Disallow: /site/olspage.jsp?id=cat12076*
21- Disallow: /site/olstemplatemapper.jsp?id=cat12076*
22- Disallow: /site/olspage.jsp?id=cat12080*
23- Disallow: /site/olstemplatemapper.jsp?id=cat12080*
24- Disallow: /site/olspage.jsp?id=cat12101*
25- Disallow: /site/olstemplatemapper.jsp?id=cat12101*
26- Disallow: /site/olstemplatemapper.jsp?id=pcat17006*
27- Disallow: /site/olspage.jsp?id=pcat17006*
2830  Disallow: /browse-api/*
29- Disallow: /site/clp/test-page/*
30- Disallow: /site/test-page/*
31- Disallow: /site/misc/test-pages/*
32- Disallow: /site/offer/*
33- Disallow: /site/compare?skus=*
3431  Disallow: /api/*.json
3532  Disallow: /availability/*
3633  Disallow: /cart
@@ -41,6 +38,19 @@
4138  Disallow: /pricing/*
4239  Disallow: /productfulfillment/*
4340  Disallow: /profile/*
41+ Disallow: /services/plans/*
42+ Disallow: /services/shoppingscheduler*
43+ Disallow: /site/builder/*
44+ Disallow: /site/modals/*
45+ Disallow: /site/dni/*
46+ Disallow: /site/offerbuilder/*
47+
48+ # User Dynamic Input Paths
49+ Disallow: /site/questions/submission*
50+ Disallow: /site/answers/submission*
51+ Disallow: /site/reviews/submission*
52+
53+ # Production-Safe Explicit Tab Fragments
4454  Disallow: /*template=_gameDetailsTab
4555  Disallow: /*template=_movieDetailsTab
4656  Disallow: /*template=_musicDetailsTab
@@ -53,39 +63,8 @@
5363  Disallow: /*template=_specificationsTab
5464  Disallow: /*template=_featuresSpoke
5565  Disallow: /*template=_buyingOptionsNewTab
56- Disallow: /site/modals/*
57- Disallow: /site/clp/conditional-offers/*
58- Disallow: /site/total-tech-support/*
59- Disallow: /site/clp/annette-test-listing/*
60- Disallow: /site/shop-view-test-page/*
61- Disallow: /site/clp/test/*
62- Disallow: /site/test-cloud-ready/*
63- Disallow: /site/clp/mystery-rewards-certificate/pcmcat1524134875052.c?id=pcmcat1524134875052
64- Disallow: /services/plans/*
65- Disallow: /site/dni/*
66- Disallow: /site/offerbuilder/*
67- Disallow: /services/shoppingscheduler*
68- Disallow: /site/test-page/*
69- Disallow: /site/questions/submission*
70- Disallow: /site/answers/submission*
71- Disallow: /site/reviews/submission*
72- Disallow: *==$
73- Disallow: */-?
74- Disallow: */undefined$
75- Disallow: */null$
76- Disallow: *javascript:winopengld*
77- Disallow: */site/olspage.jsp*
78- Disallow: */product-selector/*
79- Disallow: /purchasehistory/gift-receipt*
80- Disallow: /contact-us?*
81- Disallow: /ease/email
82- Disallow: /ihc/invoice*
83- Disallow: /identity/newAccount?*
84- Disallow: /identity/signin?*
85- Disallow: /identity/signin/forgotPassword?*
86- Disallow: /dossier/service/appointment/*/*-*
87- Disallow: /dossier/service/appointment/search?*
8866 
67+ # Production-Safe Explicit Core Facet Controls (Brand/Category left open)
8968  Disallow: *armodelformat_facet*
9069  Disallow: *awardtype_facet*
9170  Disallow: *awardwinner_facet*
@@ -176,9 +155,9 @@
176155  Disallow: *soldout_facet*
177156  Disallow: */site/promo/*?*_facet*
178157  Disallow: */product/list/*
179- Disallow: */email-subscription/*
158+ Disallow: */awacs-ingestor/*
180159 
181- #pagination
160+ # Exact Pagination Safeguards
182161  Disallow: *cp=1*
183162  Disallow: *cp=20*
184163  Disallow: *cp=6*
@@ -187,12 +166,274 @@
187166  Disallow: *cp=9*
188167  Disallow: *?*page=*&page=*$
189168 
190- #search
169+ # Search Parameters Controls
191170  Disallow: */site/searchpage.jsp?st=*
192171  Disallow: *?id*_facet*&st=*
193172  Disallow: *?id*_facet*_facet*&st=*
194173  Disallow: *?cp=*&id=*&st=*
195174 
175+ # Malformed URL Traps (Universal Protection)
176+ Disallow: *==$
177+ Disallow: */-?
178+ Disallow: */undefined$
179+ Disallow: */null$
180+ Disallow: *javascript:winopengld*
181+
182+ # ==================================================
183+ # BUCKET 2: AI TRAINING / DATA INGESTION FIREWALL
184+ # Completely blocks models from scraping copy assets for foundational training.
185+ # ==================================================
186+ User-agent: GPTBot-Web
187+ User-agent: CCBot
188+ User-agent: Anthropic-AI
189+ User-agent: Claude-Web
190+ User-agent: Bytespider
191+ Disallow: /[1.1]
192+
193+ # ==================================================
194+ # BUCKET 3: UNIFIED WEB ENGINES & UNIVERSAL DEFAULTS
195+ # Consolidated grouping ensures zero functional divergence between indexing bots.
196+ # ==================================================
197+ User-agent: Googlebot
198+ User-agent: Bingbot
199+ User-agent: DuckDuckBot
200+ User-agent: Slurp
201+ User-agent: *
202+
203+ # Standard Asset Pathways
204+ Allow: /~assets/
205+ Allow: /discover-learn/
206+
207+ # Core System Blockouts
208+ Disallow: /nex/
209+ Disallow: /m/e/*
210+ Disallow: /site/builder/*
211+ Disallow: /browse-api/*
212+ Disallow: /api/*.json
213+ Disallow: /availability/*
214+ Disallow: /cart
215+ Disallow: /cart/*
216+ Disallow: /cartitems/*
217+ Disallow: /fulfillment/*
218+ Disallow: /location/*
219+ Disallow: /pricing/*
220+ Disallow: /productfulfillment/*
221+ Disallow: /profile/*
222+ Disallow: /site/modals/*
223+ Disallow: /services/plans/*
224+ Disallow: /site/dni/*
225+ Disallow: /site/offerbuilder/*
226+ Disallow: /services/shoppingscheduler*
227+
228+ # Staging, Internal Test, & Promotional Exclusion Matrices
229+ Disallow: /site/clp/test-page/*
230+ Disallow: /site/test-page/*
231+ Disallow: /site/misc/test-pages/*
232+ Disallow: /site/offer/*
233+ Disallow: /site/compare?skus=*
234+ Disallow: /site/clp/conditional-offers/*
235+ Disallow: /site/total-tech-support/*
236+ Disallow: /site/clp/annette-test-listing/*
237+ Disallow: /site/shop-view-test-page/*
238+ Disallow: /site/clp/test/*
239+ Disallow: /site/test-cloud-ready/*
240+ Disallow: /site/clp/mystery-rewards-certificate/pcmcat1524134875052.c?id=pcmcat1524134875052
241+ Disallow: /purchasehistory/gift-receipt*
242+ Disallow: /contact-us?*
243+ Disallow: /ease/email
244+ Disallow: /ihc/invoice*
245+ Disallow: /identity/newAccount?*
246+ Disallow: /identity/signin?*
247+ Disallow: /identity/signin/forgotPassword?*
248+ Disallow: /dossier/service/appointment/*/*-*
249+ Disallow: /dossier/service/appointment/search?*
250+ Disallow: */product-selector/*
251+ Disallow: */email-subscription/*
252+
253+ # Legacy Template Controller Protections (Parameter Strictness Maintained)
254+ Disallow: /site/olspage.jsp?id=pcat17098*
255+ Disallow: /site/olstemplatemapper.jsp?id=pcat17098*
256+ Disallow: /site/olspage.jsp?id=pcat17089*
257+ Disallow: /site/olstemplatemapper.jsp?id=pcat17089*
258+ Disallow: /site/olspage.jsp?id=cat13503*
259+ Disallow: /site/olstemplatemapper.jsp?id=cat13503*
260+ Disallow: /site/olspage.jsp?id=pcat17007*
261+ Disallow: /site/olstemplatemapper.jsp?id=pcat17007*
262+ Disallow: /site/olspage.jsp?id=pcat17408*
263+ Disallow: /site/olstemplatemapper.jsp?id=pcat17408*
264+ Disallow: /site/olspage.jsp?id=cat12091*
265+ Disallow: /site/olstemplatemapper.jsp?id=cat12091*
266+ Disallow: /site/olspage.jsp?id=pcat17005*
267+ Disallow: /site/olstemplatemapper.jsp?id=pcat17005*
268+ Disallow: /site/olspage.jsp?id=cat12076*
269+ Disallow: /site/olstemplatemapper.jsp?id=cat12076*
270+ Disallow: /site/olspage.jsp?id=cat12080*
271+ Disallow: /site/olstemplatemapper.jsp?id=cat12080*
272+ Disallow: /site/olspage.jsp?id=cat12101*
273+ Disallow: /site/olstemplatemapper.jsp?id=cat12101*
274+ Disallow: /site/olstemplatemapper.jsp?id=pcat17006*
275+ Disallow: /site/olspage.jsp?id=pcat17006*
276+ Disallow: */site/olspage.jsp*
277+
278+ # Asynchronous Layout Fragment Controls
279+ Disallow: /*template=_gameDetailsTab
280+ Disallow: /*template=_movieDetailsTab
281+ Disallow: /*template=_musicDetailsTab
282+ Disallow: /*template=_softwareDetailsTab
283+ Disallow: /*template=_accessoriesTab
284+ Disallow: /*template=_castAndCrewTab
285+ Disallow: /*template=_editorialTab
286+ Disallow: /*template=_episodesTab
287+ Disallow: /*template=_protectionAndServicesTab
288+ Disallow: /*template=_specificationsTab
289+ Disallow: /*template=_featuresSpoke
290+ Disallow: /*template=_buyingOptionsNewTab
291+
292+ # User Input Paths
293+ Disallow: /site/questions/submission*
294+ Disallow: /site/answers/submission*
295+ Disallow: /site/reviews/submission*
296+
297+ # Granular Facet Exclusions
298+ Disallow: *armodelformat_facet*
299+ Disallow: *awardtype_facet*
300+ Disallow: *awardwinner_facet*
301+ Disallow: *awardyear_facet*
302+ Disallow: *blenderfunctions_facet*
303+ Disallow: *boxdesign_facet*
304+ Disallow: *cast_facet*
305+ Disallow: *cleaningpathwidthrange_facet*
306+ Disallow: *consumerlifestage_facet*
307+ Disallow: *decade_facet*
308+ Disallow: *defrosttype_facet*
309+ Disallow: *departmentid_facet*
310+ Disallow: *depthtype_facet*
311+ Disallow: *displayport_facet*
312+ Disallow: *doorstylesv_facet*
313+ Disallow: *drawertype_facet*
314+ Disallow: *dynamiccategory_facet*
315+ Disallow: *effectivemegapixels_facet*
316+ Disallow: *flashmodesmv_facet*
317+ Disallow: *framematerialmv_facet*
318+ Disallow: *giftcardthemesv_facet*
319+ Disallow: *handletype_facet*
320+ Disallow: *headsetstylesv_facet*
321+ Disallow: *heatingsource_facet*
322+ Disallow: *imagesensorsize_facet*
323+ Disallow: *imagesensortype11_facet*
324+ Disallow: *includedcolors_facet*
325+ Disallow: *inkmodelcompatibilitymv_facet*
326+ Disallow: *internalorexternal_facet*
327+ Disallow: *laptopuse_facet*
328+ Disallow: *lengthofcable_facet*
329+ Disallow: *mattressmaterial_facet*
330+ Disallow: *maximumsensitivity_facet*
331+ Disallow: *minfocallengthrange_facet*
332+ Disallow: *mobileinternalmemory_facet*
333+ Disallow: *monitorresponsetimerange_facet*
334+ Disallow: *monitorsnumberofusb30typecports_facet*
335+ Disallow: *monitorsnumberofusb31typecports_facet*
336+ Disallow: *mountmaterial_facet*
337+ Disallow: *networkingcabletype_facet*
338+ Disallow: *numberofbedroomsef_facet*
339+ Disallow: *numberofburnerssv_facet*
340+ Disallow: *numberofbuttons_facet*
341+ Disallow: *numberofdryingcycles_facet*
342+ Disallow: *numberoffanspeedssv_facet*
343+ Disallow: *numberofhdmiinputssv_facet*
344+ Disallow: *numberoflanports_facet*
345+ Disallow: *numberofspeakers_facet*
346+ Disallow: *numberofspeedssv_facet*
347+ Disallow: *numberofthunderbolt_ports_facet*
348+ Disallow: *numberofwashrinsecycle_facet*
349+ Disallow: *papersize_facet*
350+ Disallow: *playbackformatsmv_facet*
351+ Disallow: *productcategories_facet*
352+ Disallow: *screensizediagonalrange_facet*
353+ Disallow: *securityconnectivity_facet*
354+ Disallow: *securityresolution_facet*
355+ Disallow: *securitystoragetype_facet*
356+ Disallow: *softwareformat_facet*
357+ Disallow: *sqbprocessormodel_facet*
358+ Disallow: *sqbtotalstoragecapacity_facet*
359+ Disallow: *stringtype_facet*
360+ Disallow: *subclassid_facet*
361+ Disallow: *subgenre_facet*
362+ Disallow: *thruthedoordispenser_facet*
363+ Disallow: *toastingslotssv_facet*
364+ Disallow: *totaltechdeals_facet*
365+ Disallow: *touchscreentype_facet*
366+ Disallow: *transferrate_facet*
367+ Disallow: *transistorsize_facet*
368+ Disallow: *transmitterdistance_facet*
369+ Disallow: *treeheight_facet*
370+ Disallow: *treelightbulbcolor_facet*
371+ Disallow: *tubfinishmv_facet*
372+ Disallow: *varyinganglescreen_facet*
373+ Disallow: *vendorcollection_facet*
374+ Disallow: *videomemory_facet*
375+ Disallow: *viewfindertypemv_facet*
376+ Disallow: *virtualrealitycompatibility_facet*
377+ Disallow: *wanportssv_facet*
378+ Disallow: *wearingstyle_facet*
379+ Disallow: *currentprice2_facet*
380+ Disallow: *customerreviews_facet*Customer*Rating*5*
381+ Disallow: *customerreviews_facet*Customer*Rating*4*
382+ Disallow: *customerreviews_facet*Customer*Rating*3*
383+ Disallow: *customerreviews_facet*Customer*Rating*2*
384+ Disallow: *customerreviews_facet*Customer*Rating*1*
385+ Disallow: *soldout_facet*
386+ Disallow: */site/promo/*?*_facet*
387+ Disallow: */product/list/*
388+
389+ # Pagination Crawl Trap Controls
390+ Disallow: *cp=1*
391+ Disallow: *cp=20*
392+ Disallow: *cp=6*
393+ Disallow: *cp=7*
394+ Disallow: *cp=8*
395+ Disallow: *cp=9*
396+ Disallow: *?*page=*&page=*$
397+
398+ # Search Parameters Controls
399+ Disallow: */site/searchpage.jsp?st=*
400+ Disallow: *?id*_facet*&st=*
401+ Disallow: *?id*_facet*_facet*&st=*
402+ Disallow: *?cp=*&id=*&st=*
403+
404+ # Malformed URL Traps
405+ Disallow: *==$
406+ Disallow: */-?
407+ Disallow: */undefined$
408+ Disallow: */null$
409+ Disallow: *javascript:winopengld*
410+
411+ # ==================================================
412+ # BUCKET 4: PAID SEARCH / ADS / MEDIA BOTS
413+ # Leaves pathways unobstructed for digital advertising synchronization.
414+ # ==================================================
415+ User-agent: AdsBot-Google
416+ User-agent: Googlebot-Image
417+ User-agent: Mediapartners-Google
418+ User-agent: Google-Display-Ads-Bot
419+ Disallow:
420+
421+ # ==================================================
422+ # BUCKET 5: BLOCKED AGGRESSIVE THIRD-PARTY AGGREGATORS / TOOLS
423+ # Absolute block to preserve server performance and block unauthorized scrapers.
424+ # ==================================================
425+ User-agent: GetIntentCrawler
426+ User-agent: DotBot
427+ User-agent: MJ12bot
428+ User-agent: thesis-research-bot
429+ User-agent: TerraCotta
430+ User-agent: BigSurAI
431+ User-agent: MyCentralAIScraper
432+ Disallow: / [2.1]
433+
434+ # ==================================================
435+ # DATA SITEMAPS
436+ # ==================================================
196437  Sitemap: https://sitemaps.bestbuy.com/sitemaps_discover_learn.xml
197438  Sitemap: https://sitemaps.bestbuy.com/sitemaps_pdp.xml
198439  Sitemap: https://sitemaps.bestbuy.com/sitemaps_promos.xml
@@ -204,14 +445,3 @@
204445  Sitemap: https://sitemaps.bestbuy.com/sitemaps_manually_curated_pages.xml
205446  Sitemap: https://sitemaps.bestbuy.com/sitemaps_combos.xml
206447  Sitemap: https://sitemaps.bestbuy.com/sitemaps_bsin.xml
207-
208- User-agent: GetIntentCrawler
209- Disallow: /
210-
211- #Crawler specific settings
212- User-agent: AdsBot-Google
213- User-agent: Googlebot-Image
214- User-agent: Mediapartners-Google
215- Disallow:
216- User-agent: Google-Display-Ads-Bot
217- Disallow: