NeuralCrawl

Insight Enterprises / robots.txt snapshot

← back to insight.com · fetched 2026-08-27T20:19:04Z (22d ago) · HTTP 200 · 4150 bytes · sha256 9e46bb65d09cc148 · raw

final URL: https://www.insight.com/robots.txt

1# robots.txt for www.insight.com
2# Last updated: 2026-06-10
3#
4# Locale-agnostic design: Disallow rules use /* wildcards so they apply
5# across all locale prefixes (en_US, en_GB, de_DE, fr_FR, etc.) without
6# needing per-locale entries.
7#
8# Buy subdomain handled separately at buy.insight.com/robots.txt
9
10# -----------------------------------------------
11# General crawlers
12# -----------------------------------------------
13User-agent: *
14
15# Carve out specific query parameters that should remain crawlable
16Allow: /*?qtype=
17Allow: /*?pq=
18Allow: /*?identifier=shopping
19Allow: /*?partnermessage
20Allow: /insightweb/*.css$
21Allow: /*.html
22Allow: /*/shop/product/
23Allow: /*%23*
24
25# Block all other query parameters (catches session IDs, tracking,
26# print/preview flags, faceted search, etc. — the allows above take
27# precedence for whitelisted params)
28Disallow: /*?*
29
30# Block internal search result pages (duplicate/low-value content)
31Disallow: /*/search*.html
32
33# Block AEM app and system paths
34Disallow: /insightweb/
35Disallow: /flytrap/
36
37# Block AEM content/dam asset paths (PDFs, raw DAM)
38Disallow: /content/dam/insight-web/*/solutions/service-provider/microsite/assets/
39Disallow: /content/dam/insight-web/*/pdfs/
40Disallow: /content/dam/insight/
41Disallow: /content/dam/global/*/pdfs/
42
43# Block AEM content repository paths (help, client, sandbox)
44Disallow: /content/insight-web/*/help/*
45Disallow: /content/insight-web/*/client/*
46Disallow: /content/insight-web/*/Sandbox/*
47Disallow: /content/insight-web/*/sandbox/*
48
49# Block internal/admin tooling
50Disallow: /admin/
51Disallow: /cms/
52Disallow: /api/
53Disallow: /internal/
54
55# Block staging and test environments
56Disallow: /staging/
57Disallow: /test/
58Disallow: /uat/
59Disallow: /dev/
60Disallow: /sim/
61Disallow: /qa/
62
63Crawl-delay: 5
64
65# -----------------------------------------------
66# Blocked crawlers — AI training and data harvesting
67# -----------------------------------------------
68User-agent: GPTBot
69Disallow: /
70
71User-agent: ChatGPT-User
72Disallow: /
73
74User-agent: CCBot
75Disallow: /
76
77User-agent: anthropic-ai
78Disallow: /
79
80User-agent: Claude-Web
81Disallow: /
82
83User-agent: cohere-ai
84Disallow: /
85
86User-agent: Omgilibot
87Disallow: /
88
89User-agent: FacebookBot
90Disallow: /
91
92User-agent: Bytespider
93Disallow: /
94
95User-agent: NeevaAI
96Disallow: /
97
98User-agent: Firecrawl
99Disallow: /
100
101User-agent: Kadoa
102Disallow: /
103
104User-agent: ImagesiftBot
105Disallow: /
106
107# -----------------------------------------------
108# Allowed referral/discovery crawlers
109# -----------------------------------------------
110User-agent: PerplexityBot
111Allow: /
112
113# -----------------------------------------------
114# Googlebot
115# -----------------------------------------------
116User-agent: Googlebot
117
118Allow: /*?qtype=
119Allow: /*?pq=
120Allow: /*?identifier=shopping
121Allow: /*?partnermessage
122Allow: /*/shop/product/
123
124Disallow: /*?*
125Disallow: /*/search*.html
126Disallow: /*/account/
127Disallow: /*/login
128Disallow: /*/logout
129Disallow: /*/register
130Disallow: /insightweb/
131Disallow: /flytrap/
132Disallow: /content/dam/insight-web/*/pdfs/
133Disallow: /content/dam/insight/
134Disallow: /content/insight-web/*/help/*
135Disallow: /content/insight-web/*/client/*
136Disallow: /content/insight-web/*/Sandbox/*
137Disallow: /content/insight-web/*/sandbox/*
138Disallow: /admin/
139Disallow: /api/
140
141Crawl-delay: 1
142
143# -----------------------------------------------
144# Googlebot-Image
145# -----------------------------------------------
146User-agent: Googlebot-Image
147
148Disallow: /*/account/
149Disallow: /*/shop/cart
150Disallow: /*/shop/checkout
151Disallow: /content/dam/insight-web/*/pdfs/
152Disallow: /content/dam/global/*/pdfs/
153
154# -----------------------------------------------
155# Bingbot
156# -----------------------------------------------
157User-agent: bingbot
158
159Allow: /*?qtype=
160Allow: /*?pq=
161Allow: /*/shop/product/
162
163Disallow: /*?*
164Disallow: /*/search*.html
165Disallow: /*/account/
166Disallow: /*/login
167Disallow: /insightweb/
168Disallow: /flytrap/
169Disallow: /content/dam/insight-web/*/pdfs/
170Disallow: /content/dam/insight/
171Disallow: /admin/
172Disallow: /api/
173
174Crawl-delay: 2
175
176# -----------------------------------------------
177# Sitemaps
178# -----------------------------------------------
179Sitemap: https://www.insight.com/sitemap.xml