NeuralCrawl

Toyota / robots.txt snapshot

← back to toyota.com · fetched 2026-08-26T20:03:19Z (23d ago) · HTTP 200 · 12630 bytes · sha256 37b883a99db192f7 · raw

final URL: https://www.toyota.com/robots.txt

1# =============================================================================
2# robots.txt — toyota.com
3# 2026-09-01
4# =============================================================================
5
6
7# Training crawlers — blocked everywhere
8
9User-agent: GPTBot
10Disallow: /
11
12User-agent: ClaudeBot
13Disallow: /
14
15User-agent: anthropic-ai
16Disallow: /
17
18User-agent: Google-Extended
19Disallow: /
20
21User-agent: Applebot-Extended
22Disallow: /
23
24User-agent: Meta-ExternalAgent
25Disallow: /
26
27User-agent: meta-externalagent
28Disallow: /
29
30User-agent: Bytespider
31Disallow: /
32
33User-agent: CCBot
34Disallow: /
35
36User-agent: cohere-ai
37Disallow: /
38
39User-agent: Diffbot
40Disallow: /
41
42User-agent: Omgilibot
43Disallow: /
44
45User-agent: Amazonbot
46Disallow: /
47
48User-agent: DuckAssistBot
49Disallow: /
50
51
52# Stealth crawler — declarative only, real enforcement at WAF
53
54User-agent: PerplexityBot
55Disallow: /
56
57
58# User-initiated AI agents — allowed on public pages, blocked on sensitive sections
59
60User-agent: ChatGPT-User
61Disallow: /kbb*
62Disallow: /local-specials*
63Disallow: /what-fits-my-budget*
64Disallow: /dealers*
65Disallow: /payment-estimator*
66Disallow: /compare*
67Disallow: /configurator*
68Disallow: /search-inventory*
69Disallow: /request-a-quote*
70Disallow: /owners/my-vehicle/service-history*
71Disallow: /owners/vehicle-specification*
72Disallow: /owners/vehicle-dashboard*
73Disallow: /akamai/
74Disallow: /campaigns/
75Disallow: /apps/recruiting/
76Disallow: /analytics/
77Disallow: /search/
78Disallow: /toyotaSearch/
79Disallow: /byt4/pub/
80Disallow: /byt/
81Disallow: /dealer-inventory/
82Disallow: /espanol/dealer-inventory/
83Disallow: /hris/HRPolicy.htm
84Disallow: /owners/web/pages/home?p_auth=*
85Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
86Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
87Disallow: /owners/parts-service/toyota-service-care
88Disallow: /t3portal/
89Disallow: /t3Portal/
90Disallow: /responsive/
91Disallow: /stylish-cars/
92Disallow: /capable-suvs/
93Disallow: /rugged-trucks/
94Disallow: /powerful-hybrid-suvs/
95Disallow: /espanol/stylish-cars/
96Disallow: /espanol/capable-suvs/
97Disallow: /espanol/rugged-trucks/
98Disallow: /espanol/powerful-hybrid-suvs/
99Disallow: /_tcm/
100Disallow: /content/ebrochure/toyota-mobility.pdf
101Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
102Disallow: /firmware-updates/download
103Disallow: /firmware-updates/updates
104Disallow: /firmware-updates/usb-steps
105Disallow: /iframe-test
106Disallow: /resources/smartpath/
107Disallow: /espanol/resources/smartpath/
108Disallow: /resources/rct/
109Disallow: /espanol/resources/rct/
110Disallow: /configurator/*zipcode=
111Allow: /
112
113User-agent: Claude-User
114Disallow: /kbb*
115Disallow: /local-specials*
116Disallow: /what-fits-my-budget*
117Disallow: /dealers*
118Disallow: /payment-estimator*
119Disallow: /compare*
120Disallow: /configurator*
121Disallow: /search-inventory*
122Disallow: /request-a-quote*
123Disallow: /owners/my-vehicle/service-history*
124Disallow: /owners/vehicle-specification*
125Disallow: /owners/vehicle-dashboard*
126Disallow: /akamai/
127Disallow: /campaigns/
128Disallow: /apps/recruiting/
129Disallow: /analytics/
130Disallow: /search/
131Disallow: /toyotaSearch/
132Disallow: /byt4/pub/
133Disallow: /byt/
134Disallow: /dealer-inventory/
135Disallow: /espanol/dealer-inventory/
136Disallow: /hris/HRPolicy.htm
137Disallow: /owners/web/pages/home?p_auth=*
138Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
139Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
140Disallow: /owners/parts-service/toyota-service-care
141Disallow: /t3portal/
142Disallow: /t3Portal/
143Disallow: /responsive/
144Disallow: /stylish-cars/
145Disallow: /capable-suvs/
146Disallow: /rugged-trucks/
147Disallow: /powerful-hybrid-suvs/
148Disallow: /espanol/stylish-cars/
149Disallow: /espanol/capable-suvs/
150Disallow: /espanol/rugged-trucks/
151Disallow: /espanol/powerful-hybrid-suvs/
152Disallow: /_tcm/
153Disallow: /content/ebrochure/toyota-mobility.pdf
154Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
155Disallow: /firmware-updates/download
156Disallow: /firmware-updates/updates
157Disallow: /firmware-updates/usb-steps
158Disallow: /iframe-test
159Disallow: /resources/smartpath/
160Disallow: /espanol/resources/smartpath/
161Disallow: /resources/rct/
162Disallow: /espanol/resources/rct/
163Disallow: /configurator/*zipcode=
164Allow: /
165
166User-agent: Perplexity-User
167Disallow: /kbb*
168Disallow: /local-specials*
169Disallow: /what-fits-my-budget*
170Disallow: /dealers*
171Disallow: /payment-estimator*
172Disallow: /compare*
173Disallow: /configurator*
174Disallow: /search-inventory*
175Disallow: /request-a-quote*
176Disallow: /owners/my-vehicle/service-history*
177Disallow: /owners/vehicle-specification*
178Disallow: /owners/vehicle-dashboard*
179Disallow: /akamai/
180Disallow: /campaigns/
181Disallow: /apps/recruiting/
182Disallow: /analytics/
183Disallow: /search/
184Disallow: /toyotaSearch/
185Disallow: /byt4/pub/
186Disallow: /byt/
187Disallow: /dealer-inventory/
188Disallow: /espanol/dealer-inventory/
189Disallow: /hris/HRPolicy.htm
190Disallow: /owners/web/pages/home?p_auth=*
191Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
192Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
193Disallow: /owners/parts-service/toyota-service-care
194Disallow: /t3portal/
195Disallow: /t3Portal/
196Disallow: /responsive/
197Disallow: /stylish-cars/
198Disallow: /capable-suvs/
199Disallow: /rugged-trucks/
200Disallow: /powerful-hybrid-suvs/
201Disallow: /espanol/stylish-cars/
202Disallow: /espanol/capable-suvs/
203Disallow: /espanol/rugged-trucks/
204Disallow: /espanol/powerful-hybrid-suvs/
205Disallow: /_tcm/
206Disallow: /content/ebrochure/toyota-mobility.pdf
207Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
208Disallow: /firmware-updates/download
209Disallow: /firmware-updates/updates
210Disallow: /firmware-updates/usb-steps
211Disallow: /iframe-test
212Disallow: /resources/smartpath/
213Disallow: /espanol/resources/smartpath/
214Disallow: /resources/rct/
215Disallow: /espanol/resources/rct/
216Disallow: /configurator/*zipcode=
217Allow: /
218
219
220# Verified AI search and answer crawlers — same treatment as user-initiated bots
221
222User-agent: OAI-SearchBot
223Disallow: /kbb*
224Disallow: /local-specials*
225Disallow: /what-fits-my-budget*
226Disallow: /dealers*
227Disallow: /payment-estimator*
228Disallow: /compare*
229Disallow: /configurator*
230Disallow: /search-inventory*
231Disallow: /request-a-quote*
232Disallow: /owners/my-vehicle/service-history*
233Disallow: /owners/vehicle-specification*
234Disallow: /owners/vehicle-dashboard*
235Disallow: /akamai/
236Disallow: /campaigns/
237Disallow: /apps/recruiting/
238Disallow: /analytics/
239Disallow: /search/
240Disallow: /toyotaSearch/
241Disallow: /byt4/pub/
242Disallow: /byt/
243Disallow: /dealer-inventory/
244Disallow: /espanol/dealer-inventory/
245Disallow: /hris/HRPolicy.htm
246Disallow: /owners/web/pages/home?p_auth=*
247Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
248Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
249Disallow: /owners/parts-service/toyota-service-care
250Disallow: /t3portal/
251Disallow: /t3Portal/
252Disallow: /responsive/
253Disallow: /stylish-cars/
254Disallow: /capable-suvs/
255Disallow: /rugged-trucks/
256Disallow: /powerful-hybrid-suvs/
257Disallow: /espanol/stylish-cars/
258Disallow: /espanol/capable-suvs/
259Disallow: /espanol/rugged-trucks/
260Disallow: /espanol/powerful-hybrid-suvs/
261Disallow: /_tcm/
262Disallow: /content/ebrochure/toyota-mobility.pdf
263Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
264Disallow: /firmware-updates/download
265Disallow: /firmware-updates/updates
266Disallow: /firmware-updates/usb-steps
267Disallow: /iframe-test
268Disallow: /resources/smartpath/
269Disallow: /espanol/resources/smartpath/
270Disallow: /resources/rct/
271Disallow: /espanol/resources/rct/
272Disallow: /configurator/*zipcode=
273Allow: /
274
275User-agent: Claude-SearchBot
276Disallow: /kbb*
277Disallow: /local-specials*
278Disallow: /what-fits-my-budget*
279Disallow: /dealers*
280Disallow: /payment-estimator*
281Disallow: /compare*
282Disallow: /configurator*
283Disallow: /search-inventory*
284Disallow: /request-a-quote*
285Disallow: /owners/my-vehicle/service-history*
286Disallow: /owners/vehicle-specification*
287Disallow: /owners/vehicle-dashboard*
288Disallow: /akamai/
289Disallow: /campaigns/
290Disallow: /apps/recruiting/
291Disallow: /analytics/
292Disallow: /search/
293Disallow: /toyotaSearch/
294Disallow: /byt4/pub/
295Disallow: /byt/
296Disallow: /dealer-inventory/
297Disallow: /espanol/dealer-inventory/
298Disallow: /hris/HRPolicy.htm
299Disallow: /owners/web/pages/home?p_auth=*
300Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
301Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
302Disallow: /owners/parts-service/toyota-service-care
303Disallow: /t3portal/
304Disallow: /t3Portal/
305Disallow: /responsive/
306Disallow: /stylish-cars/
307Disallow: /capable-suvs/
308Disallow: /rugged-trucks/
309Disallow: /powerful-hybrid-suvs/
310Disallow: /espanol/stylish-cars/
311Disallow: /espanol/capable-suvs/
312Disallow: /espanol/rugged-trucks/
313Disallow: /espanol/powerful-hybrid-suvs/
314Disallow: /_tcm/
315Disallow: /content/ebrochure/toyota-mobility.pdf
316Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
317Disallow: /firmware-updates/download
318Disallow: /firmware-updates/updates
319Disallow: /firmware-updates/usb-steps
320Disallow: /iframe-test
321Disallow: /resources/smartpath/
322Disallow: /espanol/resources/smartpath/
323Disallow: /resources/rct/
324Disallow: /espanol/resources/rct/
325Disallow: /configurator/*zipcode=
326Allow: /
327
328
329# Applebot — verified Apple search index, separate from Applebot-Extended above
330
331User-agent: Applebot
332Disallow: /kbb*
333Disallow: /local-specials*
334Disallow: /what-fits-my-budget*
335Disallow: /dealers*
336Disallow: /payment-estimator*
337Disallow: /compare*
338Disallow: /configurator*
339Disallow: /search-inventory*
340Disallow: /request-a-quote*
341Disallow: /owners/my-vehicle/service-history*
342Disallow: /owners/vehicle-specification*
343Disallow: /owners/vehicle-dashboard*
344Disallow: /akamai/
345Disallow: /campaigns/
346Disallow: /apps/recruiting/
347Disallow: /analytics/
348Disallow: /search/
349Disallow: /toyotaSearch/
350Disallow: /byt4/pub/
351Disallow: /byt/
352Disallow: /dealer-inventory/
353Disallow: /espanol/dealer-inventory/
354Disallow: /hris/HRPolicy.htm
355Disallow: /owners/web/pages/home?p_auth=*
356Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
357Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
358Disallow: /owners/parts-service/toyota-service-care
359Disallow: /t3portal/
360Disallow: /t3Portal/
361Disallow: /responsive/
362Disallow: /stylish-cars/
363Disallow: /capable-suvs/
364Disallow: /rugged-trucks/
365Disallow: /powerful-hybrid-suvs/
366Disallow: /espanol/stylish-cars/
367Disallow: /espanol/capable-suvs/
368Disallow: /espanol/rugged-trucks/
369Disallow: /espanol/powerful-hybrid-suvs/
370Disallow: /_tcm/
371Disallow: /content/ebrochure/toyota-mobility.pdf
372Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
373Disallow: /firmware-updates/download
374Disallow: /firmware-updates/updates
375Disallow: /firmware-updates/usb-steps
376Disallow: /iframe-test
377Disallow: /resources/smartpath/
378Disallow: /espanol/resources/smartpath/
379Disallow: /resources/rct/
380Disallow: /espanol/resources/rct/
381Disallow: /configurator/*zipcode=
382Allow: /
383
384
385# Catch-all — sensitive sections closed for everything else
386
387User-agent: *
388Disallow: /kbb*
389Disallow: /local-specials*
390Disallow: /what-fits-my-budget*
391Disallow: /dealers*
392Disallow: /payment-estimator*
393Disallow: /compare*
394Disallow: /configurator*
395Disallow: /search-inventory*
396Disallow: /request-a-quote*
397Disallow: /owners/my-vehicle/service-history*
398Disallow: /owners/vehicle-specification*
399Disallow: /owners/vehicle-dashboard*
400Disallow: /akamai/
401Disallow: /campaigns/
402Disallow: /apps/recruiting/
403Disallow: /analytics/
404Disallow: /search/
405Disallow: /toyotaSearch/
406Disallow: /byt4/pub/
407Disallow: /byt/
408Disallow: /dealer-inventory/
409Disallow: /espanol/dealer-inventory/
410Disallow: /hris/HRPolicy.htm
411Disallow: /owners/web/pages/home?p_auth=*
412Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
413Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
414Disallow: /owners/parts-service/toyota-service-care
415Disallow: /t3portal/
416Disallow: /t3Portal/
417Disallow: /responsive/
418Disallow: /stylish-cars/
419Disallow: /capable-suvs/
420Disallow: /rugged-trucks/
421Disallow: /powerful-hybrid-suvs/
422Disallow: /espanol/stylish-cars/
423Disallow: /espanol/capable-suvs/
424Disallow: /espanol/rugged-trucks/
425Disallow: /espanol/powerful-hybrid-suvs/
426Disallow: /_tcm/
427Disallow: /content/ebrochure/toyota-mobility.pdf
428Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
429Disallow: /firmware-updates/download
430Disallow: /firmware-updates/updates
431Disallow: /firmware-updates/usb-steps
432Disallow: /iframe-test
433Disallow: /resources/smartpath/
434Disallow: /espanol/resources/smartpath/
435Disallow: /resources/rct/
436Disallow: /espanol/resources/rct/
437Disallow: /configurator/*zipcode=
438
439Sitemap: https://www.toyota.com/sitemap_index.xml