# Robots.txt for Insight.com
# Global policy first; specialized blocks only where rules differ.
# Precedence note: Google/Bing resolve conflicts by longest matching rule
# (Allow wins ties). Rules below are ordered for readability, not precedence.

User-agent: *

# 1. ALLOW RULES (Rendering Assets, Core Paths & Query Parameters)
Allow: /insightweb/*.css$
Allow: /insightweb/*.css?
Allow: /insightweb/*.js$
Allow: /insightweb/*.js?
Allow: /insightweb/*.svg$
Allow: /insightweb/*.svg?
Allow: /insightweb/*.woff$
Allow: /insightweb/*.woff2$

Allow: /shop/product/
Allow: /*/shop/product/

# Whitelisted query parameters (these outrank the product query-string 
# disallows below because they are longer, more specific rules)
Allow: /*qtype=
Allow: /*pq=
Allow: /*identifier=shopping
Allow: /*partnermessage=

# 2. DISALLOW RULES (System folders, Internal DAM & Query Strings)
# Blocks all parameterized query URLs globally (including /search.html?q=...)
Disallow: /*?

# Target all localized search pages specifically (with or without parameters)
Disallow: /search.html*
Disallow: /*/search.html*

# Prevent /shop/product/ allows (above) from opening ALL parameterized 
# product URLs (faceted nav, sort orders, tracking params).
# Whitelisted params still crawl via the longer Allow rules above.
Disallow: /shop/product/*?
Disallow: /*/shop/product/*?

# System Folders
Disallow: /insightweb/
Disallow: /flytrap/

# AEM Content & DAM Paths
Disallow: /content/dam/insight/
Disallow: /content/dam/insight-web/*/solutions/service-provider/microsite/assets/
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/global/*/pdfs/
Disallow: /content/dam/global/media/ppt/
Disallow: /content/dam/global/media/zip/
Disallow: /content/dam/global/*/media/
Disallow: /content/dam/global/*/clientdocs/

# Staging, Help & Sandbox Folders (Wildcard ending blocks both slash and non-slash paths)
Disallow: /content/insight-web/*/help*
Disallow: /content/insight-web/*/client*
Disallow: /content/insight-web/*/Sandbox*
Disallow: /content/insight-web/*/sandbox*

############################
# AI TRAINING CRAWLERS - BLOCKED
# Policy: block model training; remain visible in AI search/answers.
# Search & user-triggered agents (OAI-SearchBot, ChatGPT-User, 
# Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User)
# are intentionally NOT blocked - they drive citations and referrals.
############################

# OpenAI - training
User-agent: GPTBot
Disallow: /

# Anthropic - training
User-agent: ClaudeBot
Disallow: /

# Google - Gemini training/grounding opt-out (does not affect Google Search)
User-agent: Google-Extended
Disallow: /

# Apple - Apple Intelligence training opt-out (does not affect Applebot/Siri search)
User-agent: Applebot-Extended
Disallow: /

# Meta - training (current token)
User-agent: meta-externalagent
Disallow: /

# Meta - training (legacy token, kept for coverage)
User-agent: FacebookBot
Disallow: /

# Common Crawl - training corpora
User-agent: CCBot
Disallow: /

# Amazon - training
User-agent: Amazonbot
Disallow: /

# ByteDance - training. Known to ignore robots.txt; 
# enforce at CDN/WAF level as well.
User-agent: Bytespider
Disallow: /

# Perplexity - training/crawling (excludes user-triggered search citations)
User-agent: Perplexity
Disallow: /

############################
# COMMERCIAL SCRAPERS - BLOCKED
############################
User-agent: Firecrawl
Disallow: /

User-agent: Kadoa
Disallow: /

User-agent: ImagesiftBot
Disallow: /

Sitemap: https://www.insight.com/sitemap.xml