Insight Enterprises / robots.txt
← back to insight.com · change detected 22d ago (2026-08-28T02:24:20Z) · modified +9 −137
What changed
- 🤖 AIUnblocked anthropic-ai (Anthropic) - rules removed
- Removed all rules for user-agent ‘bingbot’
- 🤖 AIUnblocked ChatGPT-User (OpenAI) - rules removed
- 🤖 AIUnblocked Claude-Web (Anthropic) - rules removed
- 🤖 AIUnblocked cohere-ai (Cohere) - rules removed
- Removed all rules for user-agent ‘Googlebot’
- Removed all rules for user-agent ‘Googlebot-Image’
- 🤖 AIUnblocked GPTBot (OpenAI) - rules removed
- Removed all rules for user-agent ‘Omgilibot’
- 🤖 AIRemoved rules for AI crawler PerplexityBot (Perplexity)
- Removed Allow: /*%23* for *
- Removed Allow: /*.html for *
- Removed Disallow: /admin/ for *
- Removed Disallow: /api/ for *
- Removed Disallow: /cms/ for *
- Removed Disallow: /dev/ for *
- Removed Disallow: /internal/ for *
- Removed Disallow: /qa/ for *
- Removed Disallow: /sim/ for *
- Removed Disallow: /staging/ for *
- Removed Disallow: /test/ for *
- Removed Disallow: /uat/ for *
- Crawl-delay for *: 5 → none
Diff old (2026-08-27T20:19:04Z) → new (2026-08-28T02:24:20Z)
| @@ -1,98 +1,42 @@ | ||
| 1 | - # robots.txt for www.insight.com | |
| 2 | - # Last updated: 2026-06-10 | |
| 3 | - # | |
| 4 | - # Locale-agnostic design: Disallow rules use /* wildcards so they apply | |
| 5 | - # across all locale prefixes (en_US, en_GB, de_DE, fr_FR, etc.) without | |
| 6 | - # needing per-locale entries. | |
| 7 | - # | |
| 8 | - # Buy subdomain handled separately at buy.insight.com/robots.txt | |
| 1 | + # Robots.txt for Insight.com | |
| 2 | + # Use specialized blocks only if rules differ from the global policy. | |
| 9 | 3 | |
| 10 | - # ----------------------------------------------- | |
| 11 | - # General crawlers | |
| 12 | - # ----------------------------------------------- | |
| 13 | 4 | User-agent: * |
| 14 | - | |
| 15 | - # Carve out specific query parameters that should remain crawlable | |
| 5 | + # 1. ALLOW RULES (Specific parameters & paths) | |
| 16 | 6 | Allow: /*?qtype= |
| 17 | 7 | Allow: /*?pq= |
| 18 | 8 | Allow: /*?identifier=shopping |
| 19 | 9 | Allow: /*?partnermessage |
| 20 | 10 | Allow: /insightweb/*.css$ |
| 21 | - Allow: /*.html | |
| 22 | 11 | Allow: /*/shop/product/ |
| 23 | - Allow: /*%23* | |
| 24 | 12 | |
| 25 | - # Block all other query parameters (catches session IDs, tracking, | |
| 26 | - # print/preview flags, faceted search, etc. — the allows above take | |
| 27 | - # precedence for whitelisted params) | |
| 13 | + # 2. DISALLOW RULES (System folders and global parameters) | |
| 28 | 14 | Disallow: /*?* |
| 29 | - | |
| 30 | - # Block internal search result pages (duplicate/low-value content) | |
| 31 | 15 | Disallow: /*/search*.html |
| 32 | - | |
| 33 | - # Block AEM app and system paths | |
| 34 | 16 | Disallow: /insightweb/ |
| 35 | 17 | Disallow: /flytrap/ |
| 36 | - | |
| 37 | - # Block AEM content/dam asset paths (PDFs, raw DAM) | |
| 38 | 18 | Disallow: /content/dam/insight-web/*/solutions/service-provider/microsite/assets/ |
| 39 | 19 | Disallow: /content/dam/insight-web/*/pdfs/ |
| 40 | 20 | Disallow: /content/dam/insight/ |
| 41 | 21 | Disallow: /content/dam/global/*/pdfs/ |
| 42 | - | |
| 43 | - # Block AEM content repository paths (help, client, sandbox) | |
| 44 | 22 | Disallow: /content/insight-web/*/help/* |
| 45 | 23 | Disallow: /content/insight-web/*/client/* |
| 46 | 24 | Disallow: /content/insight-web/*/Sandbox/* |
| 47 | 25 | Disallow: /content/insight-web/*/sandbox/* |
| 48 | 26 | |
| 49 | - # Block internal/admin tooling | |
| 50 | - Disallow: /admin/ | |
| 51 | - Disallow: /cms/ | |
| 52 | - Disallow: /api/ | |
| 53 | - Disallow: /internal/ | |
| 54 | - | |
| 55 | - # Block staging and test environments | |
| 56 | - Disallow: /staging/ | |
| 57 | - Disallow: /test/ | |
| 58 | - Disallow: /uat/ | |
| 59 | - Disallow: /dev/ | |
| 60 | - Disallow: /sim/ | |
| 61 | - Disallow: /qa/ | |
| 62 | - | |
| 63 | - Crawl-delay: 5 | |
| 64 | - | |
| 65 | - # ----------------------------------------------- | |
| 66 | - # Blocked crawlers — AI training and data harvesting | |
| 67 | - # ----------------------------------------------- | |
| 68 | - User-agent: GPTBot | |
| 69 | - Disallow: / | |
| 70 | - | |
| 71 | - User-agent: ChatGPT-User | |
| 72 | - Disallow: / | |
| 73 | - | |
| 27 | + ############################ | |
| 28 | + # BLOCKED CRAWLERS | |
| 29 | + ############################ | |
| 74 | 30 | User-agent: CCBot |
| 75 | - Disallow: / | |
| 76 | - | |
| 77 | - User-agent: anthropic-ai | |
| 78 | - Disallow: / | |
| 79 | - | |
| 80 | - User-agent: Claude-Web | |
| 81 | - Disallow: / | |
| 82 | - | |
| 83 | - User-agent: cohere-ai | |
| 84 | - Disallow: / | |
| 85 | - | |
| 86 | - User-agent: Omgilibot | |
| 87 | 31 | Disallow: / |
| 88 | 32 | |
| 89 | 33 | User-agent: FacebookBot |
| 90 | 34 | Disallow: / |
| 91 | 35 | |
| 92 | - User-agent: Bytespider | |
| 36 | + User-agent: NeevaAI | |
| 93 | 37 | Disallow: / |
| 94 | 38 | |
| 95 | - User-agent: NeevaAI | |
| 39 | + User-agent: Bytespider | |
| 96 | 40 | Disallow: / |
| 97 | 41 | |
| 98 | 42 | User-agent: Firecrawl |
| @@ -104,76 +48,4 @@ | ||
| 104 | 48 | User-agent: ImagesiftBot |
| 105 | 49 | Disallow: / |
| 106 | 50 | |
| 107 | - # ----------------------------------------------- | |
| 108 | - # Allowed referral/discovery crawlers | |
| 109 | - # ----------------------------------------------- | |
| 110 | - User-agent: PerplexityBot | |
| 111 | - Allow: / | |
| 112 | - | |
| 113 | - # ----------------------------------------------- | |
| 114 | - # Googlebot | |
| 115 | - # ----------------------------------------------- | |
| 116 | - User-agent: Googlebot | |
| 117 | - | |
| 118 | - Allow: /*?qtype= | |
| 119 | - Allow: /*?pq= | |
| 120 | - Allow: /*?identifier=shopping | |
| 121 | - Allow: /*?partnermessage | |
| 122 | - Allow: /*/shop/product/ | |
| 123 | - | |
| 124 | - Disallow: /*?* | |
| 125 | - Disallow: /*/search*.html | |
| 126 | - Disallow: /*/account/ | |
| 127 | - Disallow: /*/login | |
| 128 | - Disallow: /*/logout | |
| 129 | - Disallow: /*/register | |
| 130 | - Disallow: /insightweb/ | |
| 131 | - Disallow: /flytrap/ | |
| 132 | - Disallow: /content/dam/insight-web/*/pdfs/ | |
| 133 | - Disallow: /content/dam/insight/ | |
| 134 | - Disallow: /content/insight-web/*/help/* | |
| 135 | - Disallow: /content/insight-web/*/client/* | |
| 136 | - Disallow: /content/insight-web/*/Sandbox/* | |
| 137 | - Disallow: /content/insight-web/*/sandbox/* | |
| 138 | - Disallow: /admin/ | |
| 139 | - Disallow: /api/ | |
| 140 | - | |
| 141 | - Crawl-delay: 1 | |
| 142 | - | |
| 143 | - # ----------------------------------------------- | |
| 144 | - # Googlebot-Image | |
| 145 | - # ----------------------------------------------- | |
| 146 | - User-agent: Googlebot-Image | |
| 147 | - | |
| 148 | - Disallow: /*/account/ | |
| 149 | - Disallow: /*/shop/cart | |
| 150 | - Disallow: /*/shop/checkout | |
| 151 | - Disallow: /content/dam/insight-web/*/pdfs/ | |
| 152 | - Disallow: /content/dam/global/*/pdfs/ | |
| 153 | - | |
| 154 | - # ----------------------------------------------- | |
| 155 | - # Bingbot | |
| 156 | - # ----------------------------------------------- | |
| 157 | - User-agent: bingbot | |
| 158 | - | |
| 159 | - Allow: /*?qtype= | |
| 160 | - Allow: /*?pq= | |
| 161 | - Allow: /*/shop/product/ | |
| 162 | - | |
| 163 | - Disallow: /*?* | |
| 164 | - Disallow: /*/search*.html | |
| 165 | - Disallow: /*/account/ | |
| 166 | - Disallow: /*/login | |
| 167 | - Disallow: /insightweb/ | |
| 168 | - Disallow: /flytrap/ | |
| 169 | - Disallow: /content/dam/insight-web/*/pdfs/ | |
| 170 | - Disallow: /content/dam/insight/ | |
| 171 | - Disallow: /admin/ | |
| 172 | - Disallow: /api/ | |
| 173 | - | |
| 174 | - Crawl-delay: 2 | |
| 175 | - | |
| 176 | - # ----------------------------------------------- | |
| 177 | - # Sitemaps | |
| 178 | - # ----------------------------------------------- | |
| 179 | 51 | Sitemap: https://www.insight.com/sitemap.xml |