Toyota / robots.txt

← back to toyota.com · change detected 1mo ago (2026-10-01T01:22:45Z) · modified +29 −395

What changed

Removed Disallow: /_tcm/ for *
Removed Disallow: /akamai/ for *
Removed Disallow: /byt/ for *
Removed Disallow: /byt4/pub/ for *
Removed Disallow: /capable-suvs/ for *
Removed Disallow: /configurator/*zipcode= for *
Removed Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf for *
Removed Disallow: /content/ebrochure/toyota-mobility.pdf for *
Removed Disallow: /dealer-inventory/ for *
Removed Disallow: /espanol/capable-suvs/ for *
Removed Disallow: /espanol/dealer-inventory/ for *
Removed Disallow: /espanol/powerful-hybrid-suvs/ for *
Removed Disallow: /espanol/rugged-trucks/ for *
Removed Disallow: /espanol/stylish-cars/ for *
Removed Disallow: /firmware-updates/download for *
Removed Disallow: /firmware-updates/updates for *
Removed Disallow: /firmware-updates/usb-steps for *
Removed Disallow: /hris/HRPolicy.htm for *
Removed Disallow: /iframe-test for *
Removed Disallow: /owners/my-vehicle/service-history* for *
Removed Disallow: /owners/parts-service/toyota-service-care for *
Removed Disallow: /owners/vehicle-dashboard* for *
Removed Disallow: /owners/vehicle-specification* for *
Removed Disallow: /owners/web/pages/home?p_auth=* for *
Removed Disallow: /owners/web/pages/parts-service/service-offers?p_auth=* for *
Removed Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=* for *
Removed Disallow: /powerful-hybrid-suvs/ for *
Removed Disallow: /responsive/ for *
Removed Disallow: /rugged-trucks/ for *
Removed Disallow: /search/ for *
Removed Disallow: /stylish-cars/ for *
Removed Disallow: /t3Portal/ for *
Removed Disallow: /t3portal/ for *
Removed Disallow: /toyotaSearch/ for *
Removed Allow: / for Applebot
Removed Disallow: /_tcm/ for Applebot
Removed Disallow: /akamai/ for Applebot
Removed Disallow: /byt/ for Applebot
Removed Disallow: /byt4/pub/ for Applebot
Removed Disallow: /capable-suvs/ for Applebot
Removed Disallow: /configurator/*zipcode= for Applebot
Removed Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf for Applebot
Removed Disallow: /content/ebrochure/toyota-mobility.pdf for Applebot
Removed Disallow: /dealer-inventory/ for Applebot
Removed Disallow: /espanol/capable-suvs/ for Applebot
Removed Disallow: /espanol/dealer-inventory/ for Applebot
Removed Disallow: /espanol/powerful-hybrid-suvs/ for Applebot
Removed Disallow: /espanol/rugged-trucks/ for Applebot
Removed Disallow: /espanol/stylish-cars/ for Applebot
Removed Disallow: /firmware-updates/download for Applebot
... and 194 more change(s)

Diff old (2026-08-26T20:03:19Z) → new (2026-10-01T01:22:45Z)

		@@ -1,333 +1,35 @@
1		- # =============================================================================
	1	+ # ==================================================================
2	2	# robots.txt — toyota.com
3		- # 2026-09-01
4		- # =============================================================================
	3	+ # 2026-09-15
	4	+ # =======================================================================
5	5
6		-
7		- # Training crawlers — blocked everywhere
8		-
	6	+ # GROUP 1 — Named AI Training Crawlers, including PerplexityBot — blocked everywhere
	7	+
9	8	User-agent: GPTBot
10		- Disallow: /
11		-
12	9	User-agent: ClaudeBot
13		- Disallow: /
14		-
	10	+ User-agent: Google-Extended
	11	+ User-agent: Applebot-Extended
	12	+ User-agent: Meta-ExternalAgent
	13	+ User-agent: meta-externalagent
	14	+ User-agent: Bytespider
	15	+ User-agent: CCBot
	16	+ User-agent: cohere-ai
	17	+ User-agent: Diffbot
	18	+ User-agent: Omgilibot
	19	+ User-agent: Amazonbot
	20	+ User-agent: DuckAssistBot
15	21	User-agent: anthropic-ai
16		- Disallow: /
17		-
18		- User-agent: Google-Extended
19		- Disallow: /
20		-
21		- User-agent: Applebot-Extended
22		- Disallow: /
23		-
24		- User-agent: Meta-ExternalAgent
25		- Disallow: /
26		-
27		- User-agent: meta-externalagent
28		- Disallow: /
29		-
30		- User-agent: Bytespider
31		- Disallow: /
32		-
33		- User-agent: CCBot
34		- Disallow: /
35		-
36		- User-agent: cohere-ai
37		- Disallow: /
38		-
39		- User-agent: Diffbot
40		- Disallow: /
41		-
42		- User-agent: Omgilibot
43		- Disallow: /
44		-
45		- User-agent: Amazonbot
46		- Disallow: /
47		-
48		- User-agent: DuckAssistBot
49		- Disallow: /
50		-
51		-
52		- # Stealth crawler — declarative only, real enforcement at WAF
53		-
54	22	User-agent: PerplexityBot
55	23	Disallow: /
56		-
57		-
58		- # User-initiated AI agents — allowed on public pages, blocked on sensitive sections
59		-
	24	+
	25	+
	26	+ # GROUP 2 — Named Search and user-initiated agents — allowed on public pages, blocked on sensitive sections
	27	+
60	28	User-agent: ChatGPT-User
61		- Disallow: /kbb*
62		- Disallow: /local-specials*
63		- Disallow: /what-fits-my-budget*
64		- Disallow: /dealers*
65		- Disallow: /payment-estimator*
66		- Disallow: /compare*
67		- Disallow: /configurator*
68		- Disallow: /search-inventory*
69		- Disallow: /request-a-quote*
70		- Disallow: /owners/my-vehicle/service-history*
71		- Disallow: /owners/vehicle-specification*
72		- Disallow: /owners/vehicle-dashboard*
73		- Disallow: /akamai/
74		- Disallow: /campaigns/
75		- Disallow: /apps/recruiting/
76		- Disallow: /analytics/
77		- Disallow: /search/
78		- Disallow: /toyotaSearch/
79		- Disallow: /byt4/pub/
80		- Disallow: /byt/
81		- Disallow: /dealer-inventory/
82		- Disallow: /espanol/dealer-inventory/
83		- Disallow: /hris/HRPolicy.htm
84		- Disallow: /owners/web/pages/home?p_auth=*
85		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
86		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
87		- Disallow: /owners/parts-service/toyota-service-care
88		- Disallow: /t3portal/
89		- Disallow: /t3Portal/
90		- Disallow: /responsive/
91		- Disallow: /stylish-cars/
92		- Disallow: /capable-suvs/
93		- Disallow: /rugged-trucks/
94		- Disallow: /powerful-hybrid-suvs/
95		- Disallow: /espanol/stylish-cars/
96		- Disallow: /espanol/capable-suvs/
97		- Disallow: /espanol/rugged-trucks/
98		- Disallow: /espanol/powerful-hybrid-suvs/
99		- Disallow: /_tcm/
100		- Disallow: /content/ebrochure/toyota-mobility.pdf
101		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
102		- Disallow: /firmware-updates/download
103		- Disallow: /firmware-updates/updates
104		- Disallow: /firmware-updates/usb-steps
105		- Disallow: /iframe-test
106		- Disallow: /resources/smartpath/
107		- Disallow: /espanol/resources/smartpath/
108		- Disallow: /resources/rct/
109		- Disallow: /espanol/resources/rct/
110		- Disallow: /configurator/*zipcode=
111		- Allow: /
112		-
113	29	User-agent: Claude-User
114		- Disallow: /kbb*
115		- Disallow: /local-specials*
116		- Disallow: /what-fits-my-budget*
117		- Disallow: /dealers*
118		- Disallow: /payment-estimator*
119		- Disallow: /compare*
120		- Disallow: /configurator*
121		- Disallow: /search-inventory*
122		- Disallow: /request-a-quote*
123		- Disallow: /owners/my-vehicle/service-history*
124		- Disallow: /owners/vehicle-specification*
125		- Disallow: /owners/vehicle-dashboard*
126		- Disallow: /akamai/
127		- Disallow: /campaigns/
128		- Disallow: /apps/recruiting/
129		- Disallow: /analytics/
130		- Disallow: /search/
131		- Disallow: /toyotaSearch/
132		- Disallow: /byt4/pub/
133		- Disallow: /byt/
134		- Disallow: /dealer-inventory/
135		- Disallow: /espanol/dealer-inventory/
136		- Disallow: /hris/HRPolicy.htm
137		- Disallow: /owners/web/pages/home?p_auth=*
138		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
139		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
140		- Disallow: /owners/parts-service/toyota-service-care
141		- Disallow: /t3portal/
142		- Disallow: /t3Portal/
143		- Disallow: /responsive/
144		- Disallow: /stylish-cars/
145		- Disallow: /capable-suvs/
146		- Disallow: /rugged-trucks/
147		- Disallow: /powerful-hybrid-suvs/
148		- Disallow: /espanol/stylish-cars/
149		- Disallow: /espanol/capable-suvs/
150		- Disallow: /espanol/rugged-trucks/
151		- Disallow: /espanol/powerful-hybrid-suvs/
152		- Disallow: /_tcm/
153		- Disallow: /content/ebrochure/toyota-mobility.pdf
154		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
155		- Disallow: /firmware-updates/download
156		- Disallow: /firmware-updates/updates
157		- Disallow: /firmware-updates/usb-steps
158		- Disallow: /iframe-test
159		- Disallow: /resources/smartpath/
160		- Disallow: /espanol/resources/smartpath/
161		- Disallow: /resources/rct/
162		- Disallow: /espanol/resources/rct/
163		- Disallow: /configurator/*zipcode=
164		- Allow: /
165		-
166	30	User-agent: Perplexity-User
167		- Disallow: /kbb*
168		- Disallow: /local-specials*
169		- Disallow: /what-fits-my-budget*
170		- Disallow: /dealers*
171		- Disallow: /payment-estimator*
172		- Disallow: /compare*
173		- Disallow: /configurator*
174		- Disallow: /search-inventory*
175		- Disallow: /request-a-quote*
176		- Disallow: /owners/my-vehicle/service-history*
177		- Disallow: /owners/vehicle-specification*
178		- Disallow: /owners/vehicle-dashboard*
179		- Disallow: /akamai/
180		- Disallow: /campaigns/
181		- Disallow: /apps/recruiting/
182		- Disallow: /analytics/
183		- Disallow: /search/
184		- Disallow: /toyotaSearch/
185		- Disallow: /byt4/pub/
186		- Disallow: /byt/
187		- Disallow: /dealer-inventory/
188		- Disallow: /espanol/dealer-inventory/
189		- Disallow: /hris/HRPolicy.htm
190		- Disallow: /owners/web/pages/home?p_auth=*
191		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
192		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
193		- Disallow: /owners/parts-service/toyota-service-care
194		- Disallow: /t3portal/
195		- Disallow: /t3Portal/
196		- Disallow: /responsive/
197		- Disallow: /stylish-cars/
198		- Disallow: /capable-suvs/
199		- Disallow: /rugged-trucks/
200		- Disallow: /powerful-hybrid-suvs/
201		- Disallow: /espanol/stylish-cars/
202		- Disallow: /espanol/capable-suvs/
203		- Disallow: /espanol/rugged-trucks/
204		- Disallow: /espanol/powerful-hybrid-suvs/
205		- Disallow: /_tcm/
206		- Disallow: /content/ebrochure/toyota-mobility.pdf
207		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
208		- Disallow: /firmware-updates/download
209		- Disallow: /firmware-updates/updates
210		- Disallow: /firmware-updates/usb-steps
211		- Disallow: /iframe-test
212		- Disallow: /resources/smartpath/
213		- Disallow: /espanol/resources/smartpath/
214		- Disallow: /resources/rct/
215		- Disallow: /espanol/resources/rct/
216		- Disallow: /configurator/*zipcode=
217		- Allow: /
218		-
219		-
220		- # Verified AI search and answer crawlers — same treatment as user-initiated bots
221		-
222	31	User-agent: OAI-SearchBot
223		- Disallow: /kbb*
224		- Disallow: /local-specials*
225		- Disallow: /what-fits-my-budget*
226		- Disallow: /dealers*
227		- Disallow: /payment-estimator*
228		- Disallow: /compare*
229		- Disallow: /configurator*
230		- Disallow: /search-inventory*
231		- Disallow: /request-a-quote*
232		- Disallow: /owners/my-vehicle/service-history*
233		- Disallow: /owners/vehicle-specification*
234		- Disallow: /owners/vehicle-dashboard*
235		- Disallow: /akamai/
236		- Disallow: /campaigns/
237		- Disallow: /apps/recruiting/
238		- Disallow: /analytics/
239		- Disallow: /search/
240		- Disallow: /toyotaSearch/
241		- Disallow: /byt4/pub/
242		- Disallow: /byt/
243		- Disallow: /dealer-inventory/
244		- Disallow: /espanol/dealer-inventory/
245		- Disallow: /hris/HRPolicy.htm
246		- Disallow: /owners/web/pages/home?p_auth=*
247		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
248		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
249		- Disallow: /owners/parts-service/toyota-service-care
250		- Disallow: /t3portal/
251		- Disallow: /t3Portal/
252		- Disallow: /responsive/
253		- Disallow: /stylish-cars/
254		- Disallow: /capable-suvs/
255		- Disallow: /rugged-trucks/
256		- Disallow: /powerful-hybrid-suvs/
257		- Disallow: /espanol/stylish-cars/
258		- Disallow: /espanol/capable-suvs/
259		- Disallow: /espanol/rugged-trucks/
260		- Disallow: /espanol/powerful-hybrid-suvs/
261		- Disallow: /_tcm/
262		- Disallow: /content/ebrochure/toyota-mobility.pdf
263		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
264		- Disallow: /firmware-updates/download
265		- Disallow: /firmware-updates/updates
266		- Disallow: /firmware-updates/usb-steps
267		- Disallow: /iframe-test
268		- Disallow: /resources/smartpath/
269		- Disallow: /espanol/resources/smartpath/
270		- Disallow: /resources/rct/
271		- Disallow: /espanol/resources/rct/
272		- Disallow: /configurator/*zipcode=
273		- Allow: /
274		-
275	32	User-agent: Claude-SearchBot
276		- Disallow: /kbb*
277		- Disallow: /local-specials*
278		- Disallow: /what-fits-my-budget*
279		- Disallow: /dealers*
280		- Disallow: /payment-estimator*
281		- Disallow: /compare*
282		- Disallow: /configurator*
283		- Disallow: /search-inventory*
284		- Disallow: /request-a-quote*
285		- Disallow: /owners/my-vehicle/service-history*
286		- Disallow: /owners/vehicle-specification*
287		- Disallow: /owners/vehicle-dashboard*
288		- Disallow: /akamai/
289		- Disallow: /campaigns/
290		- Disallow: /apps/recruiting/
291		- Disallow: /analytics/
292		- Disallow: /search/
293		- Disallow: /toyotaSearch/
294		- Disallow: /byt4/pub/
295		- Disallow: /byt/
296		- Disallow: /dealer-inventory/
297		- Disallow: /espanol/dealer-inventory/
298		- Disallow: /hris/HRPolicy.htm
299		- Disallow: /owners/web/pages/home?p_auth=*
300		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
301		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
302		- Disallow: /owners/parts-service/toyota-service-care
303		- Disallow: /t3portal/
304		- Disallow: /t3Portal/
305		- Disallow: /responsive/
306		- Disallow: /stylish-cars/
307		- Disallow: /capable-suvs/
308		- Disallow: /rugged-trucks/
309		- Disallow: /powerful-hybrid-suvs/
310		- Disallow: /espanol/stylish-cars/
311		- Disallow: /espanol/capable-suvs/
312		- Disallow: /espanol/rugged-trucks/
313		- Disallow: /espanol/powerful-hybrid-suvs/
314		- Disallow: /_tcm/
315		- Disallow: /content/ebrochure/toyota-mobility.pdf
316		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
317		- Disallow: /firmware-updates/download
318		- Disallow: /firmware-updates/updates
319		- Disallow: /firmware-updates/usb-steps
320		- Disallow: /iframe-test
321		- Disallow: /resources/smartpath/
322		- Disallow: /espanol/resources/smartpath/
323		- Disallow: /resources/rct/
324		- Disallow: /espanol/resources/rct/
325		- Disallow: /configurator/*zipcode=
326		- Allow: /
327		-
328		-
329		- # Applebot — verified Apple search index, separate from Applebot-Extended above
330		-
331	33	User-agent: Applebot
332	34	Disallow: /kbb*
333	35	Disallow: /local-specials*
		@@ -338,52 +40,16 @@
338	40	Disallow: /configurator*
339	41	Disallow: /search-inventory*
340	42	Disallow: /request-a-quote*
341		- Disallow: /owners/my-vehicle/service-history*
342		- Disallow: /owners/vehicle-specification*
343		- Disallow: /owners/vehicle-dashboard*
344		- Disallow: /akamai/
	43	+ Disallow: /analytics/
	44	+ Disallow: /apps/recruiting/
345	45	Disallow: /campaigns/
346		- Disallow: /apps/recruiting/
347		- Disallow: /analytics/
348		- Disallow: /search/
349		- Disallow: /toyotaSearch/
350		- Disallow: /byt4/pub/
351		- Disallow: /byt/
352		- Disallow: /dealer-inventory/
353		- Disallow: /espanol/dealer-inventory/
354		- Disallow: /hris/HRPolicy.htm
355		- Disallow: /owners/web/pages/home?p_auth=*
356		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
357		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
358		- Disallow: /owners/parts-service/toyota-service-care
359		- Disallow: /t3portal/
360		- Disallow: /t3Portal/
361		- Disallow: /responsive/
362		- Disallow: /stylish-cars/
363		- Disallow: /capable-suvs/
364		- Disallow: /rugged-trucks/
365		- Disallow: /powerful-hybrid-suvs/
366		- Disallow: /espanol/stylish-cars/
367		- Disallow: /espanol/capable-suvs/
368		- Disallow: /espanol/rugged-trucks/
369		- Disallow: /espanol/powerful-hybrid-suvs/
370		- Disallow: /_tcm/
371		- Disallow: /content/ebrochure/toyota-mobility.pdf
372		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
373		- Disallow: /firmware-updates/download
374		- Disallow: /firmware-updates/updates
375		- Disallow: /firmware-updates/usb-steps
376		- Disallow: /iframe-test
377	46	Disallow: /resources/smartpath/
378	47	Disallow: /espanol/resources/smartpath/
379	48	Disallow: /resources/rct/
380	49	Disallow: /espanol/resources/rct/
381		- Disallow: /configurator/*zipcode=
382		- Allow: /
383		-
384		-
	50	+
385	51	# Catch-all — sensitive sections closed for everything else
386		-
	52	+
387	53	User-agent: *
388	54	Disallow: /kbb*
389	55	Disallow: /local-specials*
		@@ -394,46 +60,14 @@
394	60	Disallow: /configurator*
395	61	Disallow: /search-inventory*
396	62	Disallow: /request-a-quote*
397		- Disallow: /owners/my-vehicle/service-history*
398		- Disallow: /owners/vehicle-specification*
399		- Disallow: /owners/vehicle-dashboard*
400		- Disallow: /akamai/
	63	+ Disallow: /analytics/
	64	+ Disallow: /apps/recruiting/
401	65	Disallow: /campaigns/
402		- Disallow: /apps/recruiting/
403		- Disallow: /analytics/
404		- Disallow: /search/
405		- Disallow: /toyotaSearch/
406		- Disallow: /byt4/pub/
407		- Disallow: /byt/
408		- Disallow: /dealer-inventory/
409		- Disallow: /espanol/dealer-inventory/
410		- Disallow: /hris/HRPolicy.htm
411		- Disallow: /owners/web/pages/home?p_auth=*
412		- Disallow: /owners/web/pages/parts-service/service-offers?p_auth=*
413		- Disallow: /owners/web/pages/parts-service/service-offers?p_p_auth=*
414		- Disallow: /owners/parts-service/toyota-service-care
415		- Disallow: /t3portal/
416		- Disallow: /t3Portal/
417		- Disallow: /responsive/
418		- Disallow: /stylish-cars/
419		- Disallow: /capable-suvs/
420		- Disallow: /rugged-trucks/
421		- Disallow: /powerful-hybrid-suvs/
422		- Disallow: /espanol/stylish-cars/
423		- Disallow: /espanol/capable-suvs/
424		- Disallow: /espanol/rugged-trucks/
425		- Disallow: /espanol/powerful-hybrid-suvs/
426		- Disallow: /_tcm/
427		- Disallow: /content/ebrochure/toyota-mobility.pdf
428		- Disallow: /content/dam/toyota/brochures/pdf/tcom/toyota-mobility.pdf
429		- Disallow: /firmware-updates/download
430		- Disallow: /firmware-updates/updates
431		- Disallow: /firmware-updates/usb-steps
432		- Disallow: /iframe-test
433	66	Disallow: /resources/smartpath/
434	67	Disallow: /espanol/resources/smartpath/
435	68	Disallow: /resources/rct/
436	69	Disallow: /espanol/resources/rct/
437		- Disallow: /configurator/*zipcode=
	70	+
	71	+
	72	+ Sitemap: https://www.toyota.com/sitemap_index.xml
438	73
439		- Sitemap: https://www.toyota.com/sitemap_index.xml