NeuralCrawl

General Motors / robots.txt

← back to gm.com · change detected 1mo ago (2026-10-02T20:00:12Z) · modified +133 −53

What changed

Diff old (2026-06-20T01:10:30Z) → new (2026-10-02T20:00:12Z)

@@ -1,10 +1,23 @@
1- # Allow GSA crawler
2- User-agent: gsa-crawler
3- Crawl-delay: 1
4- Disallow: /alumni
1+ # Allow Adobe Edge Optimizer
2+ User-agent: AdobeEdgeOptimize/1.0
3+ Crawl-delay: 10
54  Disallow: /*/no_search/
6- Disallow: */_jcr_content/
7- Disallow: *.js
5+ Disallow: *.modal.*
6+ Disallow: *.print.*
7+ Disallow: *.filter.*
8+
9+ # Allow AI Content Visibility Checker
10+ User-agent: Spacecat/1.0
11+ Crawl-delay: 10
12+ Disallow: /*/no_search/
13+ Disallow: *.modal.*
14+ Disallow: *.print.*
15+ Disallow: *.filter.*
16+
17+ # Allow AI Content Visibility Checker
18+ User-agent: Spacecat
19+ Crawl-delay: 10
20+ Disallow: /*/no_search/
821  Disallow: *.modal.*
922  Disallow: *.print.*
1023  Disallow: *.filter.*
@@ -13,8 +26,6 @@
1326  User-agent: Googlebot
1427  Crawl-delay: 10
1528  Disallow: /*/no_search/
16- Disallow: */_jcr_content/
17- Disallow: *.js
1829  Disallow: *.modal.*
1930  Disallow: *.print.*
2031  Disallow: *.filter.*
@@ -23,8 +34,6 @@
2334  User-agent: Google-Extended
2435  Crawl-delay: 10
2536  Disallow: /*/no_search/
26- Disallow: */_jcr_content/
27- Disallow: *.js
2837  Disallow: *.modal.*
2938  Disallow: *.print.*
3039  Disallow: *.filter.*
@@ -33,8 +42,102 @@
3342  User-agent: Googlebot-Image
3443  Crawl-delay: 10
3544  Disallow: /*/no_search/
36- Disallow: */_jcr_content/
37- Disallow: *.js
45+ Disallow: *.modal.*
46+ Disallow: *.print.*
47+ Disallow: *.filter.*
48+
49+ # Allow Google Mobile Bot
50+ User-agent: Googlebot-Mobile
51+ Crawl-delay: 10
52+ Disallow: /*/no_search/
53+ Disallow: *.modal.*
54+ Disallow: *.print.*
55+ Disallow: *.filter.*
56+
57+ # Allow Google PageRenderer Bot
58+ User-agent: Googlebot-PageRenderer
59+ Crawl-delay: 10
60+ Disallow: /*/no_search/
61+ Disallow: *.modal.*
62+ Disallow: *.print.*
63+ Disallow: *.filter.*
64+
65+ # Allow Google Web Snippet Bot
66+ User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/)
67+ Crawl-delay: 10
68+ Disallow: /*/no_search/
69+ Disallow: *.modal.*
70+ Disallow: *.print.*
71+ Disallow: *.filter.*
72+
73+ # Allow Google Messages Bot
74+ User-agent: GoogleMessages
75+ Crawl-delay: 10
76+ Disallow: /*/no_search/
77+ Disallow: *.modal.*
78+ Disallow: *.print.*
79+ Disallow: *.filter.*
80+
81+ # Allow ChatGPT-User
82+ User-agent: ChatGPT-User
83+ Crawl-delay: 10
84+ Disallow: /*/no_search/
85+ Disallow: *.modal.*
86+ Disallow: *.print.*
87+ Disallow: *.filter.*
88+
89+ # Allow GPTBot
90+ User-agent: GPTBot/1.3
91+ Crawl-delay: 10
92+ Disallow: /*/no_search/
93+ Disallow: *.modal.*
94+ Disallow: *.print.*
95+ Disallow: *.filter.*
96+
97+ # Allow OAI SearchBot
98+ User-agent: OAI-SearchBot
99+ Crawl-delay: 10
100+ Disallow: /*/no_search/
101+ Disallow: *.modal.*
102+ Disallow: *.print.*
103+ Disallow: *.filter.*
104+
105+ # Allow OAI SearchBot
106+ User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)
107+ Crawl-delay: 10
108+ Disallow: /*/no_search/
109+ Disallow: *.modal.*
110+ Disallow: *.print.*
111+ Disallow: *.filter.*
112+
113+ # Allow Claude SearchBot
114+ User-agent: Claude-SearchBot
115+ Crawl-delay: 10
116+ Disallow: /*/no_search/
117+ Disallow: *.modal.*
118+ Disallow: *.print.*
119+ Disallow: *.filter.*
120+
121+ # Allow PerplexityBot
122+ User-agent: PerplexityBot
123+ Crawl-delay: 10
124+ Disallow: /*/no_search/
125+ Disallow: *.modal.*
126+ Disallow: *.print.*
127+ Disallow: *.filter.*
128+
129+ # Allow PerplexityBot
130+ User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)
131+ Crawl-delay: 10
132+ Disallow: /*/no_search/
133+ Disallow: *.modal.*
134+ Disallow: *.print.*
135+ Disallow: *.filter.*
136+
137+ # Allow Perplexity AI Assistant User Agent
138+ User-agent: Perplexity
139+ Crawl-delay: 10
140+ Disallow: /*/no_search/
38141  Disallow: *.modal.*
39142  Disallow: *.print.*
40143  Disallow: *.filter.*
@@ -43,8 +146,14 @@
43146  User-agent: Bingbot
44147  Crawl-delay: 10
45148  Disallow: /*/no_search/
46- Disallow: */_jcr_content/
47- Disallow: *.js
149+ Disallow: *.modal.*
150+ Disallow: *.print.*
151+ Disallow: *.filter.*
152+
153+ # Allow Bing Bot
154+ User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
155+ Crawl-delay: 10
156+ Disallow: /*/no_search/
48157  Disallow: *.modal.*
49158  Disallow: *.print.*
50159  Disallow: *.filter.*
@@ -53,8 +162,6 @@
53162  User-agent: msnbot
54163  Crawl-delay: 10
55164  Disallow: /*/no_search/
56- Disallow: */_jcr_content/
57- Disallow: *.js
58165  Disallow: *.modal.*
59166  Disallow: *.print.*
60167  Disallow: *.filter.*
@@ -63,8 +170,6 @@
63170  User-agent: Applebot
64171  Crawl-delay: 10
65172  Disallow: /*/no_search/
66- Disallow: */_jcr_content/
67- Disallow: *.js
68173  Disallow: *.modal.*
69174  Disallow: *.print.*
70175  Disallow: *.filter.*
@@ -73,8 +178,6 @@
73178  User-agent: DuckDuckBot
74179  Crawl-delay: 10
75180  Disallow: /*/no_search/
76- Disallow: */_jcr_content/
77- Disallow: *.js
78181  Disallow: *.modal.*
79182  Disallow: *.print.*
80183  Disallow: *.filter.*
@@ -83,8 +186,6 @@
83186  User-Agent: Slurp
84187  Crawl-delay: 10
85188  Disallow: /*/no_search/
86- Disallow: */_jcr_content/
87- Disallow: *.js
88189  Disallow: *.modal.*
89190  Disallow: *.print.*
90191  Disallow: *.filter.*
@@ -93,8 +194,6 @@
93194  User-agent: facebookexternalhit
94195  Crawl-delay: 10
95196  Disallow: /*/no_search/
96- Disallow: */_jcr_content/
97- Disallow: *.js
98197  Disallow: *.modal.*
99198  Disallow: *.print.*
100199  Disallow: *.filter.*
@@ -103,8 +202,6 @@
103202  User-agent: Twitterbot
104203  Crawl-delay: 10
105204  Disallow: /*/no_search/
106- Disallow: */_jcr_content/
107- Disallow: *.js
108205  Disallow: *.modal.*
109206  Disallow: *.print.*
110207  Disallow: *.filter.*
@@ -113,8 +210,6 @@
113210  User-agent: LinkedInBot
114211  Crawl-delay: 10
115212  Disallow: /*/no_search/
116- Disallow: */_jcr_content/
117- Disallow: *.js
118213  Disallow: *.modal.*
119214  Disallow: *.print.*
120215  Disallow: *.filter.*
@@ -123,8 +218,6 @@
123218  User-agent: Baiduspider
124219  Crawl-delay: 10
125220  Disallow: /*/no_search/
126- Disallow: */_jcr_content/
127- Disallow: *.js
128221  Disallow: *.modal.*
129222  Disallow: *.print.*
130223  Disallow: *.filter.*
@@ -133,8 +226,6 @@
133226  User-agent: Bluesky Cardyb
134227  Crawl-delay: 10
135228  Disallow: /*/no_search/
136- Disallow: */_jcr_content/
137- Disallow: *.js
138229  Disallow: *.modal.*
139230  Disallow: *.print.*
140231  Disallow: *.filter.*
@@ -143,8 +234,6 @@
143234  User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bluesky Cardyb; +mailto:[email protected]) Chrome/W.X.Y.Z Safari/537.36
144235  Crawl-delay: 10
145236  Disallow: /*/no_search/
146- Disallow: */_jcr_content/
147- Disallow: *.js
148237  Disallow: *.modal.*
149238  Disallow: *.print.*
150239  Disallow: *.filter.*
@@ -153,8 +242,6 @@
153242  User-agent: SiteimproveBot
154243  Crawl-delay: 10
155244  Disallow: /*/no_search/
156- Disallow: */_jcr_content/
157- Disallow: *.js
158245  Disallow: *.modal.*
159246  Disallow: *.print.*
160247  Disallow: *.filter.*
@@ -163,8 +250,6 @@
163250  User-agent: SiteimproveBot-Crawler
164251  Crawl-delay: 10
165252  Disallow: /*/no_search/
166- Disallow: */_jcr_content/
167- Disallow: *.js
168253  Disallow: *.modal.*
169254  Disallow: *.print.*
170255  Disallow: *.filter.*
@@ -173,8 +258,6 @@
173258  User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com
174259  Crawl-delay: 10
175260  Disallow: /*/no_search/
176- Disallow: */_jcr_content/
177- Disallow: *.js
178261  Disallow: *.modal.*
179262  Disallow: *.print.*
180263  Disallow: *.filter.*
@@ -183,8 +266,6 @@
183266  User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
184267  Crawl-delay: 10
185268  Disallow: /*/no_search/
186- Disallow: */_jcr_content/
187- Disallow: *.js
188269  Disallow: *.modal.*
189270  Disallow: *.print.*
190271  Disallow: *.filter.*
@@ -193,8 +274,6 @@
193274  User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Image size by Siteimprove.com
194275  Crawl-delay: 10
195276  Disallow: /*/no_search/
196- Disallow: */_jcr_content/
197- Disallow: *.js
198277  Disallow: *.modal.*
199278  Disallow: *.print.*
200279  Disallow: *.filter.*
@@ -203,8 +282,6 @@
203282  User-agent: CSS Validator: Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0
204283  Crawl-delay: 10
205284  Disallow: /*/no_search/
206- Disallow: */_jcr_content/
207- Disallow: *.js
208285  Disallow: *.modal.*
209286  Disallow: *.print.*
210287  Disallow: *.filter.*
@@ -213,8 +290,6 @@
213290  User-agent: AhrefsBot
214291  Crawl-delay: 10
215292  Disallow: /*/no_search/
216- Disallow: */_jcr_content/
217- Disallow: *.js
218293  Disallow: *.modal.*
219294  Disallow: *.print.*
220295  Disallow: *.filter.*
@@ -223,8 +298,6 @@
223298  User-agent: Screaming Frog SEO Spider
224299  Crawl-delay: 10
225300  Disallow: /*/no_search/
226- Disallow: */_jcr_content/
227- Disallow: *.js
228301  Disallow: *.modal.*
229302  Disallow: *.print.*
230303  Disallow: *.filter.*
@@ -233,8 +306,15 @@
233306  User-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0
234307  Crawl-delay: 10
235308  Disallow: /*/no_search/
236- Disallow: */_jcr_content/
237- Disallow: *.js
309+ Disallow: *.modal.*
310+ Disallow: *.print.*
311+ Disallow: *.filter.*
312+
313+ # Allow GSA crawler
314+ User-agent: gsa-crawler
315+ Crawl-delay: 1
316+ Disallow: /alumni
317+ Disallow: /*/no_search/
238318  Disallow: *.modal.*
239319  Disallow: *.print.*
240320  Disallow: *.filter.*
@@ -245,4 +325,4 @@
245325  Disallow: /
246326 
247327 
248- # v040225d
328+ # v100226a