Tesla / robots.txt snapshot

← back to tesla.com · fetched 2026-10-20T02:14:13Z (18d ago) · HTTP 200 · 3087 bytes · sha256 04bfbc5f7de84619 · raw

final URL: https://www.tesla.com/robots.txt

1	#
2	# robots.txt
3	#
4	# This file is to prevent the crawling and indexing of certain parts
5	# of your site by web crawlers and spiders run by sites like Yahoo!
6	# and Google. By telling these "robots" where not to go on your site,
7	# you save bandwidth and server resources.
8	#
9	# This file will be ignored unless it is at the root of your host:
10	# Used: http://example.com/robots.txt
11	# Ignored: http://example.com/site/robots.txt
12	#
13	# For more information about the robots.txt standard, see:
14	# http://www.robotstxt.org/robotstxt.html
15
16	User-agent: *
17	Crawl-delay: 10
18	# Sitemap
19	Allow: /sitemap.xml
20	# CSS, JS, Images
21	Allow: /misc/*.css$
22	Allow: /misc/*.css?
23	Allow: /misc/*.js$
24	Allow: /misc/*.js?
25	Allow: /misc/*.gif
26	Allow: /misc/*.jpg
27	Allow: /misc/*.jpeg
28	Allow: /misc/*.png
29	Allow: /modules/*.css$
30	Allow: /modules/*.css?
31	Allow: /modules/*.js$
32	Allow: /modules/*.js?
33	Allow: /modules/*.gif
34	Allow: /modules/*.jpg
35	Allow: /modules/*.jpeg
36	Allow: /modules/*.png
37	Allow: /profiles/*.css$
38	Allow: /profiles/*.css?
39	Allow: /profiles/*.js$
40	Allow: /profiles/*.js?
41	Allow: /profiles/*.gif
42	Allow: /profiles/*.jpg
43	Allow: /profiles/*.jpeg
44	Allow: /profiles/*.png
45	Allow: /themes/*.css$
46	Allow: /themes/*.css?
47	Allow: /themes/*.js$
48	Allow: /themes/*.js?
49	Allow: /themes/*.gif
50	Allow: /themes/*.jpg
51	Allow: /themes/*.jpeg
52	Allow: /themes/*.png
53	# IMCE
54	Disallow: /sites/default/files/
55	# Directories
56	Disallow: /includes/
57	Disallow: /misc/
58	Disallow: /modules/
59	Disallow: /profiles/
60	Disallow: /scripts/
61	Disallow: /themes/
62	# Files
63	Disallow: /CHANGELOG.txt
64	Disallow: /cron.php
65	Disallow: /INSTALL.mysql.txt
66	Disallow: /INSTALL.pgsql.txt
67	Disallow: /INSTALL.sqlite.txt
68	Disallow: /install.php
69	Disallow: /INSTALL.txt
70	Disallow: /LICENSE.txt
71	Disallow: /MAINTAINERS.txt
72	Disallow: /update.php
73	Disallow: /UPGRADE.txt
74	Disallow: /xmlrpc.php
75	# Paths (clean URLs)
76	Disallow: /admin/
77	Disallow: /comment/reply/
78	Disallow: /filter/tips/
79	Disallow: /node/add/
80	Disallow: /search/
81	Disallow: /user/register/
82	Disallow: /user/password/
83	Disallow: /user/login/
84	Disallow: /user/logout/
85	# Paths (no clean URLs)
86	Disallow: /?q=admin/
87	Disallow: /?q=comment/reply/
88	Disallow: /?q=filter/tips/
89	Disallow: /?q=node/add/
90	Disallow: /?q=search/
91	Disallow: /?q=user/password/
92	Disallow: /?q=user/register/
93	Disallow: /?q=user/login/
94	Disallow: /?q=user/logout/
95	Disallow: /?q=user/q=node/
96
97	# Disallow indexing of Node pages
98	Disallow: /node/
99	Disallow: /*/node/
100
101	Disallow: /?q=node/
102	Disallow: /?q=*/node/
103
104	# Disallow indexing of nodequeue pages
105	Disallow: /nodequeue/
106	Disallow: /*/nodequeue/
107
108	# Disallow indexing of pages with referral
109	Disallow: /*?referral=
110	Disallow: /?referral=
111
112	##############################
113	# START TESLA CONTENT.
114	##############################
115
116	# Tesla content landing pages.
117	Disallow: /content/*
118	Disallow: /new/*
119	Disallow: /used/*
120	Disallow: /preowned/*
121	Disallow: /demo/*
122	Disallow: /taxonomy/term/*
123	Disallow: //taxonomy/term/
124	Allow: /content/dam/tesla-site/
125
126	# No indexing of en_eu
127	User-agent: Googlebot
128	Disallow: /en_eu/
129	Disallow: /en_EU/
130
131	##############################
132	# STOP TESLA CONTENT.
133	##############################
134
135	Sitemap: https://www.tesla.com/sitemap.xml