NeuralCrawl

Arrow Electronics / robots.txt snapshot

← back to arrow.com · fetched 2026-10-08T10:00:34Z (30d ago) · HTTP 200 · 2881 bytes · sha256 aeb62fadab6b3f31 · raw

final URL: https://www.arrow.com/robots.txt

1User-agent: *
2Disallow: */cart$
3Disallow: /checkout
4Disallow: */sitecore/
5
6Disallow: /akamai/akamai-sure-route.html
7
8Disallow: */reference-designs/search
9Disallow: */design-center/my-projects
10Disallow: /technical-content/
11
12#Commenting all below lines to allow crawling of URLs with query paramters
13#Disallow: *?region*
14#Disallow: *utm_source=*
15#Disallow: *%3Fregion*
16
17Disallow: */datasheets/
18Disallow: */login*
19#Disallow: */arrow-dxp/language-masters/
20#Disallow: /content/arrow-dxp/
21
22Disallow: /~/media/technical-content/
23Disallow: */my-account
24
25Disallow: */indiegogo/te-connectivity
26
27Disallow: /marketingtracking/
28Disallow: /_bm/
29Disallow: /samtec-product-selector
30Disallow: /apps/samtec/configurator/sfm
31
32Disallow: /en/register
33Disallow: /es/register
34Disallow: /de/register
35Disallow: /fr/register
36Disallow: /it/register
37Disallow: /ko/register
38Disallow: /zh/register
39Disallow: /ja/register
40
41Disallow: */shellpages
42Disallow: */errors
43
44User-agent: anthropic-ai
45Disallow: /
46Allow: /ais/*
47Allow: /globalecs/*
48
49User-agent: ClaudeBot
50Disallow: /
51Allow: /ais/*
52Allow: /globalecs/*
53
54User-agent: Google-Extended
55Disallow: /
56Allow: /ais/*
57Allow: /globalecs/*
58
59User-agent: GPTBot
60Disallow: /
61Allow: /ais/*
62Allow: /globalecs/*
63
64User-agent: ChatGPT-User
65Disallow: /
66Allow: /ais/*
67Allow: /globalecs/*
68
69User-agent: CCBot
70Disallow: /
71Allow: /ais/*
72Allow: /globalecs/*
73
74User-agent: PerplexityBot
75Disallow: /
76Allow: /ais/*
77Allow: /globalecs/*
78
79Sitemap: https://www.arrow.com/digital-content/sitemap-index.xml
80
81Sitemap: https://www.arrow.com/s-tech/post-sitemap.xml
82Sitemap: https://www.arrow.com/s-tech/page-sitemap.xml
83Sitemap: https://www.arrow.com/s-tech/category-sitemap.xml
84Sitemap: https://www.arrow.com/s-tech/post_tag-sitemap.xml
85
86Sitemap: https://www.arrow.com/ecs/ma/sitemap
87Sitemap: https://www.arrow.com/ecs/il/sitemap
88Sitemap: https://www.arrow.com/ecs/na/sitemap
89Sitemap: https://www.arrow.com/ecs/au/sitemap
90
91Sitemap: https://www.arrow.com/ecs/nz/sitemap
92Sitemap: https://www.arrow.com/ecs/at/sitemap
93Sitemap: https://www.arrow.com/ecs/ee/sitemap
94
95Sitemap: https://www.arrow.com/ecs/cz/sitemap/
96Sitemap: https://www.arrow.com/ecs/dk/sitemap/
97Sitemap: https://www.arrow.com/ecs/de/sitemap/
98
99Sitemap: https://www.arrow.com/ecs/fi/sitemap/
100Sitemap: https://www.arrow.com/ecs/fr/sitemap/
101Sitemap: https://www.arrow.com/ecs/it/sitemap/
102
103Sitemap: https://www.arrow.com/ecs/ie/sitemap/
104Sitemap: https://www.arrow.com/ecs/nl/sitemap/
105Sitemap: https://www.arrow.com/ecs/no/sitemap/
106
107Sitemap: https://www.arrow.com/ecs/pt/sitemap/
108Sitemap: https://www.arrow.com/ecs/es/sitemap/
109Sitemap: https://www.arrow.com/ecs/se/sitemap/
110
111Sitemap: https://www.arrow.com/ecs/ch-en/sitemap/
112Sitemap: https://www.arrow.com/ecs/ch-de/sitemap/
113Sitemap: https://www.arrow.com/ecs/uk/sitemap/
114
115Sitemap: https://www.arrow.com/ais/sitemap_index.xml
116Sitemap: https://www.arrow.com/ais/msembedded/sitemap_index.xml