NeuralCrawl

General Motors / robots.txt snapshot

← back to gm.com · fetched 2026-10-02T20:00:12Z (1mo ago) · HTTP 200 · 7160 bytes · sha256 6bb0e924033939d0 · raw

final URL: https://www.gm.com/robots.txt

1# Allow Adobe Edge Optimizer
2User-agent: AdobeEdgeOptimize/1.0
3Crawl-delay: 10
4Disallow: /*/no_search/
5Disallow: *.modal.*
6Disallow: *.print.*
7Disallow: *.filter.*
8
9# Allow AI Content Visibility Checker
10User-agent: Spacecat/1.0
11Crawl-delay: 10
12Disallow: /*/no_search/
13Disallow: *.modal.*
14Disallow: *.print.*
15Disallow: *.filter.*
16
17# Allow AI Content Visibility Checker
18User-agent: Spacecat
19Crawl-delay: 10
20Disallow: /*/no_search/
21Disallow: *.modal.*
22Disallow: *.print.*
23Disallow: *.filter.*
24
25# Allow Google Bot
26User-agent: Googlebot
27Crawl-delay: 10
28Disallow: /*/no_search/
29Disallow: *.modal.*
30Disallow: *.print.*
31Disallow: *.filter.*
32
33# Allow Google Extended Bot
34User-agent: Google-Extended
35Crawl-delay: 10
36Disallow: /*/no_search/
37Disallow: *.modal.*
38Disallow: *.print.*
39Disallow: *.filter.*
40
41# Allow Google Image Bot
42User-agent: Googlebot-Image
43Crawl-delay: 10
44Disallow: /*/no_search/
45Disallow: *.modal.*
46Disallow: *.print.*
47Disallow: *.filter.*
48
49# Allow Google Mobile Bot
50User-agent: Googlebot-Mobile
51Crawl-delay: 10
52Disallow: /*/no_search/
53Disallow: *.modal.*
54Disallow: *.print.*
55Disallow: *.filter.*
56
57# Allow Google PageRenderer Bot
58User-agent: Googlebot-PageRenderer
59Crawl-delay: 10
60Disallow: /*/no_search/
61Disallow: *.modal.*
62Disallow: *.print.*
63Disallow: *.filter.*
64
65# Allow Google Web Snippet Bot
66User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/)
67Crawl-delay: 10
68Disallow: /*/no_search/
69Disallow: *.modal.*
70Disallow: *.print.*
71Disallow: *.filter.*
72
73# Allow Google Messages Bot
74User-agent: GoogleMessages
75Crawl-delay: 10
76Disallow: /*/no_search/
77Disallow: *.modal.*
78Disallow: *.print.*
79Disallow: *.filter.*
80
81# Allow ChatGPT-User
82User-agent: ChatGPT-User
83Crawl-delay: 10
84Disallow: /*/no_search/
85Disallow: *.modal.*
86Disallow: *.print.*
87Disallow: *.filter.*
88
89# Allow GPTBot
90User-agent: GPTBot/1.3
91Crawl-delay: 10
92Disallow: /*/no_search/
93Disallow: *.modal.*
94Disallow: *.print.*
95Disallow: *.filter.*
96
97# Allow OAI SearchBot
98User-agent: OAI-SearchBot
99Crawl-delay: 10
100Disallow: /*/no_search/
101Disallow: *.modal.*
102Disallow: *.print.*
103Disallow: *.filter.*
104
105# Allow OAI SearchBot
106User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)
107Crawl-delay: 10
108Disallow: /*/no_search/
109Disallow: *.modal.*
110Disallow: *.print.*
111Disallow: *.filter.*
112
113# Allow Claude SearchBot
114User-agent: Claude-SearchBot
115Crawl-delay: 10
116Disallow: /*/no_search/
117Disallow: *.modal.*
118Disallow: *.print.*
119Disallow: *.filter.*
120
121# Allow PerplexityBot
122User-agent: PerplexityBot
123Crawl-delay: 10
124Disallow: /*/no_search/
125Disallow: *.modal.*
126Disallow: *.print.*
127Disallow: *.filter.*
128
129# Allow PerplexityBot
130User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)
131Crawl-delay: 10
132Disallow: /*/no_search/
133Disallow: *.modal.*
134Disallow: *.print.*
135Disallow: *.filter.*
136
137# Allow Perplexity AI Assistant User Agent
138User-agent: Perplexity
139Crawl-delay: 10
140Disallow: /*/no_search/
141Disallow: *.modal.*
142Disallow: *.print.*
143Disallow: *.filter.*
144
145# Allow Bing Bot
146User-agent: Bingbot
147Crawl-delay: 10
148Disallow: /*/no_search/
149Disallow: *.modal.*
150Disallow: *.print.*
151Disallow: *.filter.*
152
153# Allow Bing Bot
154User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
155Crawl-delay: 10
156Disallow: /*/no_search/
157Disallow: *.modal.*
158Disallow: *.print.*
159Disallow: *.filter.*
160
161# Allow MSN Bot
162User-agent: msnbot
163Crawl-delay: 10
164Disallow: /*/no_search/
165Disallow: *.modal.*
166Disallow: *.print.*
167Disallow: *.filter.*
168
169# Allow Apple Bot
170User-agent: Applebot
171Crawl-delay: 10
172Disallow: /*/no_search/
173Disallow: *.modal.*
174Disallow: *.print.*
175Disallow: *.filter.*
176
177# Allow DuckDuckGo Bot
178User-agent: DuckDuckBot
179Crawl-delay: 10
180Disallow: /*/no_search/
181Disallow: *.modal.*
182Disallow: *.print.*
183Disallow: *.filter.*
184
185# Allow Yahoo
186User-Agent: Slurp
187Crawl-delay: 10
188Disallow: /*/no_search/
189Disallow: *.modal.*
190Disallow: *.print.*
191Disallow: *.filter.*
192
193# Allow Facebook Bot
194User-agent: facebookexternalhit
195Crawl-delay: 10
196Disallow: /*/no_search/
197Disallow: *.modal.*
198Disallow: *.print.*
199Disallow: *.filter.*
200
201# Allow Twitterbot
202User-agent: Twitterbot
203Crawl-delay: 10
204Disallow: /*/no_search/
205Disallow: *.modal.*
206Disallow: *.print.*
207Disallow: *.filter.*
208
209# Allow LinkedIn Bot
210User-agent: LinkedInBot
211Crawl-delay: 10
212Disallow: /*/no_search/
213Disallow: *.modal.*
214Disallow: *.print.*
215Disallow: *.filter.*
216
217# Allow Baidu Bot
218User-agent: Baiduspider
219Crawl-delay: 10
220Disallow: /*/no_search/
221Disallow: *.modal.*
222Disallow: *.print.*
223Disallow: *.filter.*
224
225# Allow BlueSky
226User-agent: Bluesky Cardyb
227Crawl-delay: 10
228Disallow: /*/no_search/
229Disallow: *.modal.*
230Disallow: *.print.*
231Disallow: *.filter.*
232
233# Allow BlueSky
234User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bluesky Cardyb; +mailto:[email protected]) Chrome/W.X.Y.Z Safari/537.36
235Crawl-delay: 10
236Disallow: /*/no_search/
237Disallow: *.modal.*
238Disallow: *.print.*
239Disallow: *.filter.*
240
241# Allow SiteimproveBot
242User-agent: SiteimproveBot
243Crawl-delay: 10
244Disallow: /*/no_search/
245Disallow: *.modal.*
246Disallow: *.print.*
247Disallow: *.filter.*
248
249# Allow SiteimproveBot-Crawler
250User-agent: SiteimproveBot-Crawler
251Crawl-delay: 10
252Disallow: /*/no_search/
253Disallow: *.modal.*
254Disallow: *.print.*
255Disallow: *.filter.*
256
257# Allow SiteImprove SiteCheck
258User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com
259Crawl-delay: 10
260Disallow: /*/no_search/
261Disallow: *.modal.*
262Disallow: *.print.*
263Disallow: *.filter.*
264
265# Allow SiteImprove LinkCheck
266User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
267Crawl-delay: 10
268Disallow: /*/no_search/
269Disallow: *.modal.*
270Disallow: *.print.*
271Disallow: *.filter.*
272
273# Allow SiteImprove Image size
274User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Image size by Siteimprove.com
275Crawl-delay: 10
276Disallow: /*/no_search/
277Disallow: *.modal.*
278Disallow: *.print.*
279Disallow: *.filter.*
280
281# Allow SiteImprove CSS Validator
282User-agent: CSS Validator: Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0
283Crawl-delay: 10
284Disallow: /*/no_search/
285Disallow: *.modal.*
286Disallow: *.print.*
287Disallow: *.filter.*
288
289# Allow AHREFS Bot
290User-agent: AhrefsBot
291Crawl-delay: 10
292Disallow: /*/no_search/
293Disallow: *.modal.*
294Disallow: *.print.*
295Disallow: *.filter.*
296
297# Allow Screaming Frog Spider
298User-agent: Screaming Frog SEO Spider
299Crawl-delay: 10
300Disallow: /*/no_search/
301Disallow: *.modal.*
302Disallow: *.print.*
303Disallow: *.filter.*
304
305# Allow BlueSky
306User-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0
307Crawl-delay: 10
308Disallow: /*/no_search/
309Disallow: *.modal.*
310Disallow: *.print.*
311Disallow: *.filter.*
312
313# Allow GSA crawler
314User-agent: gsa-crawler
315Crawl-delay: 1
316Disallow: /alumni
317Disallow: /*/no_search/
318Disallow: *.modal.*
319Disallow: *.print.*
320Disallow: *.filter.*
321
322# Disallow MegaIndex
323User-agent: megaindex
324Crawl-delay: 10
325Disallow: /
326
327
328# v100226a