# robots.txt for www.insight.com
# Last updated: 2026-06-10
#
# Locale-agnostic design: Disallow rules use /* wildcards so they apply
# across all locale prefixes (en_US, en_GB, de_DE, fr_FR, etc.) without
# needing per-locale entries.
#
# Buy subdomain handled separately at buy.insight.com/robots.txt

# -----------------------------------------------
# General crawlers
# -----------------------------------------------
User-agent: *

# Carve out specific query parameters that should remain crawlable
Allow: /*?qtype=
Allow: /*?pq=
Allow: /*?identifier=shopping
Allow: /*?partnermessage
Allow: /insightweb/*.css$
Allow: /*.html
Allow: /*/shop/product/
Allow: /*%23*

# Block all other query parameters (catches session IDs, tracking,
# print/preview flags, faceted search, etc. — the allows above take
# precedence for whitelisted params)
Disallow: /*?*

# Block internal search result pages (duplicate/low-value content)
Disallow: /*/search*.html

# Block AEM app and system paths
Disallow: /insightweb/
Disallow: /flytrap/

# Block AEM content/dam asset paths (PDFs, raw DAM)
Disallow: /content/dam/insight-web/*/solutions/service-provider/microsite/assets/
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/insight/
Disallow: /content/dam/global/*/pdfs/

# Block AEM content repository paths (help, client, sandbox)
Disallow: /content/insight-web/*/help/*
Disallow: /content/insight-web/*/client/*
Disallow: /content/insight-web/*/Sandbox/*
Disallow: /content/insight-web/*/sandbox/*

# Block internal/admin tooling
Disallow: /admin/
Disallow: /cms/
Disallow: /api/
Disallow: /internal/

# Block staging and test environments
Disallow: /staging/
Disallow: /test/
Disallow: /uat/
Disallow: /dev/
Disallow: /sim/
Disallow: /qa/

Crawl-delay: 5

# -----------------------------------------------
# Blocked crawlers — AI training and data harvesting
# -----------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: NeevaAI
Disallow: /

User-agent: Firecrawl
Disallow: /

User-agent: Kadoa
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# -----------------------------------------------
# Allowed referral/discovery crawlers
# -----------------------------------------------
User-agent: PerplexityBot
Allow: /

# -----------------------------------------------
# Googlebot
# -----------------------------------------------
User-agent: Googlebot

Allow: /*?qtype=
Allow: /*?pq=
Allow: /*?identifier=shopping
Allow: /*?partnermessage
Allow: /*/shop/product/

Disallow: /*?*
Disallow: /*/search*.html
Disallow: /*/account/
Disallow: /*/login
Disallow: /*/logout
Disallow: /*/register
Disallow: /insightweb/
Disallow: /flytrap/
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/insight/
Disallow: /content/insight-web/*/help/*
Disallow: /content/insight-web/*/client/*
Disallow: /content/insight-web/*/Sandbox/*
Disallow: /content/insight-web/*/sandbox/*
Disallow: /admin/
Disallow: /api/

Crawl-delay: 1

# -----------------------------------------------
# Googlebot-Image
# -----------------------------------------------
User-agent: Googlebot-Image

Disallow: /*/account/
Disallow: /*/shop/cart
Disallow: /*/shop/checkout
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/global/*/pdfs/

# -----------------------------------------------
# Bingbot
# -----------------------------------------------
User-agent: bingbot

Allow: /*?qtype=
Allow: /*?pq=
Allow: /*/shop/product/

Disallow: /*?*
Disallow: /*/search*.html
Disallow: /*/account/
Disallow: /*/login
Disallow: /insightweb/
Disallow: /flytrap/
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/insight/
Disallow: /admin/
Disallow: /api/

Crawl-delay: 2

# -----------------------------------------------
# Sitemaps
# -----------------------------------------------
Sitemap: https://www.insight.com/sitemap.xml