NeuralCrawl

General Motors / robots.txt snapshot

← back to gm.com · fetched 2026-06-20T01:10:30Z (17h ago) · HTTP 200 · 5207 bytes · sha256 73fd4494ea4a0abc · raw

final URL: https://www.gm.com/robots.txt

1# Allow GSA crawler
2User-agent: gsa-crawler
3Crawl-delay: 1
4Disallow: /alumni
5Disallow: /*/no_search/
6Disallow: */_jcr_content/
7Disallow: *.js
8Disallow: *.modal.*
9Disallow: *.print.*
10Disallow: *.filter.*
11
12# Allow Google Bot
13User-agent: Googlebot
14Crawl-delay: 10
15Disallow: /*/no_search/
16Disallow: */_jcr_content/
17Disallow: *.js
18Disallow: *.modal.*
19Disallow: *.print.*
20Disallow: *.filter.*
21
22# Allow Google Extended Bot
23User-agent: Google-Extended
24Crawl-delay: 10
25Disallow: /*/no_search/
26Disallow: */_jcr_content/
27Disallow: *.js
28Disallow: *.modal.*
29Disallow: *.print.*
30Disallow: *.filter.*
31
32# Allow Google Image Bot
33User-agent: Googlebot-Image
34Crawl-delay: 10
35Disallow: /*/no_search/
36Disallow: */_jcr_content/
37Disallow: *.js
38Disallow: *.modal.*
39Disallow: *.print.*
40Disallow: *.filter.*
41
42# Allow Bing Bot
43User-agent: Bingbot
44Crawl-delay: 10
45Disallow: /*/no_search/
46Disallow: */_jcr_content/
47Disallow: *.js
48Disallow: *.modal.*
49Disallow: *.print.*
50Disallow: *.filter.*
51
52# Allow MSN Bot
53User-agent: msnbot
54Crawl-delay: 10
55Disallow: /*/no_search/
56Disallow: */_jcr_content/
57Disallow: *.js
58Disallow: *.modal.*
59Disallow: *.print.*
60Disallow: *.filter.*
61
62# Allow Apple Bot
63User-agent: Applebot
64Crawl-delay: 10
65Disallow: /*/no_search/
66Disallow: */_jcr_content/
67Disallow: *.js
68Disallow: *.modal.*
69Disallow: *.print.*
70Disallow: *.filter.*
71
72# Allow DuckDuckGo Bot
73User-agent: DuckDuckBot
74Crawl-delay: 10
75Disallow: /*/no_search/
76Disallow: */_jcr_content/
77Disallow: *.js
78Disallow: *.modal.*
79Disallow: *.print.*
80Disallow: *.filter.*
81
82# Allow Yahoo
83User-Agent: Slurp
84Crawl-delay: 10
85Disallow: /*/no_search/
86Disallow: */_jcr_content/
87Disallow: *.js
88Disallow: *.modal.*
89Disallow: *.print.*
90Disallow: *.filter.*
91
92# Allow Facebook Bot
93User-agent: facebookexternalhit
94Crawl-delay: 10
95Disallow: /*/no_search/
96Disallow: */_jcr_content/
97Disallow: *.js
98Disallow: *.modal.*
99Disallow: *.print.*
100Disallow: *.filter.*
101
102# Allow Twitterbot
103User-agent: Twitterbot
104Crawl-delay: 10
105Disallow: /*/no_search/
106Disallow: */_jcr_content/
107Disallow: *.js
108Disallow: *.modal.*
109Disallow: *.print.*
110Disallow: *.filter.*
111
112# Allow LinkedIn Bot
113User-agent: LinkedInBot
114Crawl-delay: 10
115Disallow: /*/no_search/
116Disallow: */_jcr_content/
117Disallow: *.js
118Disallow: *.modal.*
119Disallow: *.print.*
120Disallow: *.filter.*
121
122# Allow Baidu Bot
123User-agent: Baiduspider
124Crawl-delay: 10
125Disallow: /*/no_search/
126Disallow: */_jcr_content/
127Disallow: *.js
128Disallow: *.modal.*
129Disallow: *.print.*
130Disallow: *.filter.*
131
132# Allow BlueSky
133User-agent: Bluesky Cardyb
134Crawl-delay: 10
135Disallow: /*/no_search/
136Disallow: */_jcr_content/
137Disallow: *.js
138Disallow: *.modal.*
139Disallow: *.print.*
140Disallow: *.filter.*
141
142# Allow BlueSky
143User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bluesky Cardyb; +mailto:[email protected]) Chrome/W.X.Y.Z Safari/537.36
144Crawl-delay: 10
145Disallow: /*/no_search/
146Disallow: */_jcr_content/
147Disallow: *.js
148Disallow: *.modal.*
149Disallow: *.print.*
150Disallow: *.filter.*
151
152# Allow SiteimproveBot
153User-agent: SiteimproveBot
154Crawl-delay: 10
155Disallow: /*/no_search/
156Disallow: */_jcr_content/
157Disallow: *.js
158Disallow: *.modal.*
159Disallow: *.print.*
160Disallow: *.filter.*
161
162# Allow SiteimproveBot-Crawler
163User-agent: SiteimproveBot-Crawler
164Crawl-delay: 10
165Disallow: /*/no_search/
166Disallow: */_jcr_content/
167Disallow: *.js
168Disallow: *.modal.*
169Disallow: *.print.*
170Disallow: *.filter.*
171
172# Allow SiteImprove SiteCheck
173User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com
174Crawl-delay: 10
175Disallow: /*/no_search/
176Disallow: */_jcr_content/
177Disallow: *.js
178Disallow: *.modal.*
179Disallow: *.print.*
180Disallow: *.filter.*
181
182# Allow SiteImprove LinkCheck
183User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
184Crawl-delay: 10
185Disallow: /*/no_search/
186Disallow: */_jcr_content/
187Disallow: *.js
188Disallow: *.modal.*
189Disallow: *.print.*
190Disallow: *.filter.*
191
192# Allow SiteImprove Image size
193User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Image size by Siteimprove.com
194Crawl-delay: 10
195Disallow: /*/no_search/
196Disallow: */_jcr_content/
197Disallow: *.js
198Disallow: *.modal.*
199Disallow: *.print.*
200Disallow: *.filter.*
201
202# Allow SiteImprove CSS Validator
203User-agent: CSS Validator: Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0
204Crawl-delay: 10
205Disallow: /*/no_search/
206Disallow: */_jcr_content/
207Disallow: *.js
208Disallow: *.modal.*
209Disallow: *.print.*
210Disallow: *.filter.*
211
212# Allow AHREFS Bot
213User-agent: AhrefsBot
214Crawl-delay: 10
215Disallow: /*/no_search/
216Disallow: */_jcr_content/
217Disallow: *.js
218Disallow: *.modal.*
219Disallow: *.print.*
220Disallow: *.filter.*
221
222# Allow Screaming Frog Spider
223User-agent: Screaming Frog SEO Spider
224Crawl-delay: 10
225Disallow: /*/no_search/
226Disallow: */_jcr_content/
227Disallow: *.js
228Disallow: *.modal.*
229Disallow: *.print.*
230Disallow: *.filter.*
231
232# Allow BlueSky
233User-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0
234Crawl-delay: 10
235Disallow: /*/no_search/
236Disallow: */_jcr_content/
237Disallow: *.js
238Disallow: *.modal.*
239Disallow: *.print.*
240Disallow: *.filter.*
241
242# Disallow MegaIndex
243User-agent: megaindex
244Crawl-delay: 10
245Disallow: /
246
247
248# v040225d