NeuralCrawl

Toyota / robots.txt snapshot

← back to toyota.com · fetched 2026-06-20T14:27:25Z (7h ago) · HTTP 200 · 12980 bytes · sha256 02dc67deeb336b64 · raw

final URL: https://www.toyota.com/robots.txt

1# =============================================================================
2# robots.txt — toyota.com
3# 2026-05-20
4# =============================================================================
5
6
7# Training crawlers — blocked everywhere
8
9User-agent: GPTBot
10Disallow: /
11
12User-agent: ClaudeBot
13Disallow: /
14
15User-agent: anthropic-ai
16Disallow: /
17
18User-agent: Google-Extended
19Disallow: /
20
21User-agent: Applebot-Extended
22Disallow: /
23
24User-agent: Meta-ExternalAgent
25Disallow: /
26
27User-agent: meta-externalagent
28Disallow: /
29
30User-agent: Bytespider
31Disallow: /
32
33User-agent: CCBot
34Disallow: /
35
36User-agent: cohere-ai
37Disallow: /
38
39User-agent: Diffbot
40Disallow: /
41
42User-agent: Omgilibot
43Disallow: /
44
45User-agent: Amazonbot
46Disallow: /
47
48User-agent: DuckAssistBot
49Disallow: /
50
51
52# Stealth crawler — declarative only, real enforcement at WAF
53
54User-agent: PerplexityBot
55Disallow: /
56
57
58# User-initiated AI agents — allowed on public pages, blocked on sensitive sections
59
60User-agent: ChatGPT-User
61Disallow: /kbb*
62Disallow: /local-specials*
63Disallow: /what-fits-my-budget*
64Disallow: /dealers*
65Disallow: /payment-estimator*
66Disallow: /compare*
67Disallow: /configurator*
68Disallow: /search-inventory*
69Disallow: /request-a-quote*
70Disallow: /owners/my-vehicle/service-history*
71Disallow: /owners/vehicle-specification*
72Disallow: /owners/vehicle-dashboard*
73Disallow: /akamai/
74Disallow: /campaigns/
75Disallow: /apps/recruiting/
76Disallow: /analytics/
77Disallow: /search/
78Disallow: /toyotaSearch/
79Disallow: /byt4/pub/
80Disallow: /byt/
81Disallow: /dealer-inventory/
82Disallow: /espanol/dealer-inventory/
83Disallow: /hris/HRPolicy.htm
84Disallow: /owners/web/pages/home?p_auth=*
85Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
86Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
87Disallow: /owners/parts-service/toyota-service-care
88Disallow: /t3portal/
89Disallow: /t3Portal/
90Disallow: /responsive/
91Disallow: /stylish-cars/
92Disallow: /capable-suvs/
93Disallow: /rugged-trucks/
94Disallow: /powerful-hybrid-suvs/
95Disallow: /espanol/stylish-cars/
96Disallow: /espanol/capable-suvs/
97Disallow: /espanol/rugged-trucks/
98Disallow: /espanol/powerful-hybrid-suvs/
99Disallow: /_tcm/
100Disallow: /content/ebrochure/toyota-mobility.pdf
101Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
102Disallow: /firmware-updates/download
103Disallow: /firmware-updates/updates
104Disallow: /firmware-updates/usb-steps
105Disallow: /iframe-test
106Disallow: /resources/smartpath/
107Disallow: /espanol/resources/smartpath/
108Disallow: /resources/rct/
109Disallow: /espanol/resources/rct/
110Disallow: /vehicles/
111Disallow: /espanol/vehicles/
112Disallow: /configurator/*zipcode=
113Allow: /
114
115User-agent: Claude-User
116Disallow: /kbb*
117Disallow: /local-specials*
118Disallow: /what-fits-my-budget*
119Disallow: /dealers*
120Disallow: /payment-estimator*
121Disallow: /compare*
122Disallow: /configurator*
123Disallow: /search-inventory*
124Disallow: /request-a-quote*
125Disallow: /owners/my-vehicle/service-history*
126Disallow: /owners/vehicle-specification*
127Disallow: /owners/vehicle-dashboard*
128Disallow: /akamai/
129Disallow: /campaigns/
130Disallow: /apps/recruiting/
131Disallow: /analytics/
132Disallow: /search/
133Disallow: /toyotaSearch/
134Disallow: /byt4/pub/
135Disallow: /byt/
136Disallow: /dealer-inventory/
137Disallow: /espanol/dealer-inventory/
138Disallow: /hris/HRPolicy.htm
139Disallow: /owners/web/pages/home?p_auth=*
140Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
141Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
142Disallow: /owners/parts-service/toyota-service-care
143Disallow: /t3portal/
144Disallow: /t3Portal/
145Disallow: /responsive/
146Disallow: /stylish-cars/
147Disallow: /capable-suvs/
148Disallow: /rugged-trucks/
149Disallow: /powerful-hybrid-suvs/
150Disallow: /espanol/stylish-cars/
151Disallow: /espanol/capable-suvs/
152Disallow: /espanol/rugged-trucks/
153Disallow: /espanol/powerful-hybrid-suvs/
154Disallow: /_tcm/
155Disallow: /content/ebrochure/toyota-mobility.pdf
156Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
157Disallow: /firmware-updates/download
158Disallow: /firmware-updates/updates
159Disallow: /firmware-updates/usb-steps
160Disallow: /iframe-test
161Disallow: /resources/smartpath/
162Disallow: /espanol/resources/smartpath/
163Disallow: /resources/rct/
164Disallow: /espanol/resources/rct/
165Disallow: /vehicles/
166Disallow: /espanol/vehicles/
167Disallow: /configurator/*zipcode=
168Allow: /
169
170User-agent: Perplexity-User
171Disallow: /kbb*
172Disallow: /local-specials*
173Disallow: /what-fits-my-budget*
174Disallow: /dealers*
175Disallow: /payment-estimator*
176Disallow: /compare*
177Disallow: /configurator*
178Disallow: /search-inventory*
179Disallow: /request-a-quote*
180Disallow: /owners/my-vehicle/service-history*
181Disallow: /owners/vehicle-specification*
182Disallow: /owners/vehicle-dashboard*
183Disallow: /akamai/
184Disallow: /campaigns/
185Disallow: /apps/recruiting/
186Disallow: /analytics/
187Disallow: /search/
188Disallow: /toyotaSearch/
189Disallow: /byt4/pub/
190Disallow: /byt/
191Disallow: /dealer-inventory/
192Disallow: /espanol/dealer-inventory/
193Disallow: /hris/HRPolicy.htm
194Disallow: /owners/web/pages/home?p_auth=*
195Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
196Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
197Disallow: /owners/parts-service/toyota-service-care
198Disallow: /t3portal/
199Disallow: /t3Portal/
200Disallow: /responsive/
201Disallow: /stylish-cars/
202Disallow: /capable-suvs/
203Disallow: /rugged-trucks/
204Disallow: /powerful-hybrid-suvs/
205Disallow: /espanol/stylish-cars/
206Disallow: /espanol/capable-suvs/
207Disallow: /espanol/rugged-trucks/
208Disallow: /espanol/powerful-hybrid-suvs/
209Disallow: /_tcm/
210Disallow: /content/ebrochure/toyota-mobility.pdf
211Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
212Disallow: /firmware-updates/download
213Disallow: /firmware-updates/updates
214Disallow: /firmware-updates/usb-steps
215Disallow: /iframe-test
216Disallow: /resources/smartpath/
217Disallow: /espanol/resources/smartpath/
218Disallow: /resources/rct/
219Disallow: /espanol/resources/rct/
220Disallow: /vehicles/
221Disallow: /espanol/vehicles/
222Disallow: /configurator/*zipcode=
223Allow: /
224
225
226# Verified AI search and answer crawlers — same treatment as user-initiated bots
227
228User-agent: OAI-SearchBot
229Disallow: /kbb*
230Disallow: /local-specials*
231Disallow: /what-fits-my-budget*
232Disallow: /dealers*
233Disallow: /payment-estimator*
234Disallow: /compare*
235Disallow: /configurator*
236Disallow: /search-inventory*
237Disallow: /request-a-quote*
238Disallow: /owners/my-vehicle/service-history*
239Disallow: /owners/vehicle-specification*
240Disallow: /owners/vehicle-dashboard*
241Disallow: /akamai/
242Disallow: /campaigns/
243Disallow: /apps/recruiting/
244Disallow: /analytics/
245Disallow: /search/
246Disallow: /toyotaSearch/
247Disallow: /byt4/pub/
248Disallow: /byt/
249Disallow: /dealer-inventory/
250Disallow: /espanol/dealer-inventory/
251Disallow: /hris/HRPolicy.htm
252Disallow: /owners/web/pages/home?p_auth=*
253Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
254Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
255Disallow: /owners/parts-service/toyota-service-care
256Disallow: /t3portal/
257Disallow: /t3Portal/
258Disallow: /responsive/
259Disallow: /stylish-cars/
260Disallow: /capable-suvs/
261Disallow: /rugged-trucks/
262Disallow: /powerful-hybrid-suvs/
263Disallow: /espanol/stylish-cars/
264Disallow: /espanol/capable-suvs/
265Disallow: /espanol/rugged-trucks/
266Disallow: /espanol/powerful-hybrid-suvs/
267Disallow: /_tcm/
268Disallow: /content/ebrochure/toyota-mobility.pdf
269Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
270Disallow: /firmware-updates/download
271Disallow: /firmware-updates/updates
272Disallow: /firmware-updates/usb-steps
273Disallow: /iframe-test
274Disallow: /resources/smartpath/
275Disallow: /espanol/resources/smartpath/
276Disallow: /resources/rct/
277Disallow: /espanol/resources/rct/
278Disallow: /vehicles/
279Disallow: /espanol/vehicles/
280Disallow: /configurator/*zipcode=
281Allow: /
282
283User-agent: Claude-SearchBot
284Disallow: /kbb*
285Disallow: /local-specials*
286Disallow: /what-fits-my-budget*
287Disallow: /dealers*
288Disallow: /payment-estimator*
289Disallow: /compare*
290Disallow: /configurator*
291Disallow: /search-inventory*
292Disallow: /request-a-quote*
293Disallow: /owners/my-vehicle/service-history*
294Disallow: /owners/vehicle-specification*
295Disallow: /owners/vehicle-dashboard*
296Disallow: /akamai/
297Disallow: /campaigns/
298Disallow: /apps/recruiting/
299Disallow: /analytics/
300Disallow: /search/
301Disallow: /toyotaSearch/
302Disallow: /byt4/pub/
303Disallow: /byt/
304Disallow: /dealer-inventory/
305Disallow: /espanol/dealer-inventory/
306Disallow: /hris/HRPolicy.htm
307Disallow: /owners/web/pages/home?p_auth=*
308Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
309Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
310Disallow: /owners/parts-service/toyota-service-care
311Disallow: /t3portal/
312Disallow: /t3Portal/
313Disallow: /responsive/
314Disallow: /stylish-cars/
315Disallow: /capable-suvs/
316Disallow: /rugged-trucks/
317Disallow: /powerful-hybrid-suvs/
318Disallow: /espanol/stylish-cars/
319Disallow: /espanol/capable-suvs/
320Disallow: /espanol/rugged-trucks/
321Disallow: /espanol/powerful-hybrid-suvs/
322Disallow: /_tcm/
323Disallow: /content/ebrochure/toyota-mobility.pdf
324Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
325Disallow: /firmware-updates/download
326Disallow: /firmware-updates/updates
327Disallow: /firmware-updates/usb-steps
328Disallow: /iframe-test
329Disallow: /resources/smartpath/
330Disallow: /espanol/resources/smartpath/
331Disallow: /resources/rct/
332Disallow: /espanol/resources/rct/
333Disallow: /vehicles/
334Disallow: /espanol/vehicles/
335Disallow: /configurator/*zipcode=
336Allow: /
337
338
339# Applebot — verified Apple search index, separate from Applebot-Extended above
340
341User-agent: Applebot
342Disallow: /kbb*
343Disallow: /local-specials*
344Disallow: /what-fits-my-budget*
345Disallow: /dealers*
346Disallow: /payment-estimator*
347Disallow: /compare*
348Disallow: /configurator*
349Disallow: /search-inventory*
350Disallow: /request-a-quote*
351Disallow: /owners/my-vehicle/service-history*
352Disallow: /owners/vehicle-specification*
353Disallow: /owners/vehicle-dashboard*
354Disallow: /akamai/
355Disallow: /campaigns/
356Disallow: /apps/recruiting/
357Disallow: /analytics/
358Disallow: /search/
359Disallow: /toyotaSearch/
360Disallow: /byt4/pub/
361Disallow: /byt/
362Disallow: /dealer-inventory/
363Disallow: /espanol/dealer-inventory/
364Disallow: /hris/HRPolicy.htm
365Disallow: /owners/web/pages/home?p_auth=*
366Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
367Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
368Disallow: /owners/parts-service/toyota-service-care
369Disallow: /t3portal/
370Disallow: /t3Portal/
371Disallow: /responsive/
372Disallow: /stylish-cars/
373Disallow: /capable-suvs/
374Disallow: /rugged-trucks/
375Disallow: /powerful-hybrid-suvs/
376Disallow: /espanol/stylish-cars/
377Disallow: /espanol/capable-suvs/
378Disallow: /espanol/rugged-trucks/
379Disallow: /espanol/powerful-hybrid-suvs/
380Disallow: /_tcm/
381Disallow: /content/ebrochure/toyota-mobility.pdf
382Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
383Disallow: /firmware-updates/download
384Disallow: /firmware-updates/updates
385Disallow: /firmware-updates/usb-steps
386Disallow: /iframe-test
387Disallow: /resources/smartpath/
388Disallow: /espanol/resources/smartpath/
389Disallow: /resources/rct/
390Disallow: /espanol/resources/rct/
391Disallow: /vehicles/
392Disallow: /espanol/vehicles/
393Disallow: /configurator/*zipcode=
394Allow: /
395
396
397# Catch-all — sensitive sections closed for everything else
398
399User-agent: *
400Disallow: /kbb*
401Disallow: /local-specials*
402Disallow: /what-fits-my-budget*
403Disallow: /dealers*
404Disallow: /payment-estimator*
405Disallow: /compare*
406Disallow: /configurator*
407Disallow: /search-inventory*
408Disallow: /request-a-quote*
409Disallow: /owners/my-vehicle/service-history*
410Disallow: /owners/vehicle-specification*
411Disallow: /owners/vehicle-dashboard*
412Disallow: /akamai/
413Disallow: /campaigns/
414Disallow: /apps/recruiting/
415Disallow: /analytics/
416Disallow: /search/
417Disallow: /toyotaSearch/
418Disallow: /byt4/pub/
419Disallow: /byt/
420Disallow: /dealer-inventory/
421Disallow: /espanol/dealer-inventory/
422Disallow: /hris/HRPolicy.htm
423Disallow: /owners/web/pages/home?p_auth=*
424Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
425Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
426Disallow: /owners/parts-service/toyota-service-care
427Disallow: /t3portal/
428Disallow: /t3Portal/
429Disallow: /responsive/
430Disallow: /stylish-cars/
431Disallow: /capable-suvs/
432Disallow: /rugged-trucks/
433Disallow: /powerful-hybrid-suvs/
434Disallow: /espanol/stylish-cars/
435Disallow: /espanol/capable-suvs/
436Disallow: /espanol/rugged-trucks/
437Disallow: /espanol/powerful-hybrid-suvs/
438Disallow: /_tcm/
439Disallow: /content/ebrochure/toyota-mobility.pdf
440Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
441Disallow: /firmware-updates/download
442Disallow: /firmware-updates/updates
443Disallow: /firmware-updates/usb-steps
444Disallow: /iframe-test
445Disallow: /resources/smartpath/
446Disallow: /espanol/resources/smartpath/
447Disallow: /resources/rct/
448Disallow: /espanol/resources/rct/
449Disallow: /vehicles/
450Disallow: /espanol/vehicles/
451Disallow: /configurator/*zipcode=
452
453Sitemap: https://www.toyota.com/sitemap_index.xml