United States (USA.gov) / robots.txt snapshot

← back to usa.gov · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 1558 bytes · sha256 57210da4611cc41b · raw

final URL: https://www.usa.gov/robots.txt

1	#
2	# robots.txt
3	#
4	# This file is to prevent the crawling and indexing of certain parts
5	# of your site by web crawlers and spiders run by sites like Yahoo!
6	# and Google. By telling these "robots" where not to go on your site,
7	# you save bandwidth and server resources.
8	#
9	# This file will be ignored unless it is at the root of your host:
10	# Used: http://example.com/robots.txt
11	# Ignored: http://example.com/site/robots.txt
12	#
13	# For more information about the robots.txt standard, see:
14	# http://www.robotstxt.org/robotstxt.html
15
16	User-agent: *
17	Crawl-delay: 10
18
19	# Sitemaps
20	Sitemap: https://www.usa.gov/sitemap.xml
21
22	# CSS, JS, Images
23	Allow: /misc/*.css$
24	Allow: /misc/*.css?
25	Allow: /misc/*.js$
26	Allow: /misc/*.js?
27	Allow: /misc/*.gif
28	Allow: /misc/*.jpg
29	Allow: /misc/*.jpeg
30	Allow: /misc/*.png
31	Allow: /modules/*.css$
32	Allow: /modules/*.css?
33	Allow: /modules/*.js$
34	Allow: /modules/*.js?
35	Allow: /modules/*.gif
36	Allow: /modules/*.jpg
37	Allow: /modules/*.jpeg
38	Allow: /modules/*.png
39	Allow: /profiles/*.css$
40	Allow: /profiles/*.css?
41	Allow: /profiles/*.js$
42	Allow: /profiles/*.js?
43	Allow: /profiles/*.gif
44	Allow: /profiles/*.jpg
45	Allow: /profiles/*.jpeg
46	Allow: /profiles/*.png
47	Allow: /themes/*.css$
48	Allow: /themes/*.css?
49	Allow: /themes/*.js$
50	Allow: /themes/*.js?
51	Allow: /themes/*.gif
52	Allow: /themes/*.jpg
53	Allow: /themes/*.jpeg
54	Allow: /themes/*.png
55	# Directories
56	Disallow: /includes/
57	Disallow: /misc/
58	Disallow: /modules/
59	Disallow: /profiles/
60	Disallow: /scripts/
61	Disallow: /themes/
62	# We shouldn't have these, but:
63	Disallow: /node
64	Disallow: /node/
65	Disallow: /es/node
66	Disallow: /es/node/