Warner Music Group / robots.txt snapshot

← back to wmg.com · fetched 2026-06-20T01:10:31Z (15h ago) · HTTP 200 · 2783 bytes · sha256 dd38573bbab90acd · raw

final URL: https://wmg.com/robots.txt

1	#
2	# robots.txt
3	#
4	# This file is to prevent the crawling and indexing of certain parts
5	# of your site by web crawlers and spiders run by sites like Yahoo!
6	# and Google. By telling these "robots" where not to go on your site,
7	# you save bandwidth and server resources.
8	#
9	# This file will be ignored unless it is at the root of your host:
10	# Used: http://example.com/robots.txt
11	# Ignored: http://example.com/site/robots.txt
12	#
13	# For more information about the robots.txt standard, see:
14	# http://www.robotstxt.org/robotstxt.html
15
16	User-agent: *
17	Crawl-delay: 10
18	# CSS, JS, Images
19	Allow: /misc/*.css$
20	Allow: /misc/*.css?
21	Allow: /misc/*.js$
22	Allow: /misc/*.js?
23	Allow: /misc/*.gif
24	Allow: /misc/*.jpg
25	Allow: /misc/*.jpeg
26	Allow: /misc/*.png
27	Allow: /modules/*.css$
28	Allow: /modules/*.css?
29	Allow: /modules/*.js$
30	Allow: /modules/*.js?
31	Allow: /modules/*.gif
32	Allow: /modules/*.jpg
33	Allow: /modules/*.jpeg
34	Allow: /modules/*.png
35	Allow: /profiles/*.css$
36	Allow: /profiles/*.css?
37	Allow: /profiles/*.js$
38	Allow: /profiles/*.js?
39	Allow: /profiles/*.gif
40	Allow: /profiles/*.jpg
41	Allow: /profiles/*.jpeg
42	Allow: /profiles/*.png
43	Allow: /themes/*.css$
44	Allow: /themes/*.css?
45	Allow: /themes/*.js$
46	Allow: /themes/*.js?
47	Allow: /themes/*.gif
48	Allow: /themes/*.jpg
49	Allow: /themes/*.jpeg
50	Allow: /themes/*.png
51	# Directories
52	Disallow: /includes/
53	Disallow: /misc/
54	Disallow: /modules/
55	Disallow: /profiles/
56	Disallow: /scripts/
57	Disallow: /themes/
58	# Files
59	Disallow: /CHANGELOG.txt
60	Disallow: /cron.php
61	Disallow: /INSTALL.mysql.txt
62	Disallow: /INSTALL.pgsql.txt
63	Disallow: /INSTALL.sqlite.txt
64	Disallow: /install.php
65	Disallow: /INSTALL.txt
66	Disallow: /LICENSE.txt
67	Disallow: /MAINTAINERS.txt
68	Disallow: /update.php
69	Disallow: /UPGRADE.txt
70	Disallow: /xmlrpc.php
71	# Paths (clean URLs)
72	Disallow: /admin/
73	Disallow: /comment/reply/
74	Disallow: /filter/tips/
75	Disallow: /node/add/
76	Disallow: /search/
77	Disallow: /user/register/
78	Disallow: /user/password/
79	Disallow: /user/login/
80	Disallow: /user/logout/
81	Disallow: /member/
82	# Paths (no clean URLs)
83	Disallow: /?q=admin/
84	Disallow: /?q=comment/reply/
85	Disallow: /?q=filter/tips/
86	Disallow: /?q=node/add/
87	Disallow: /?q=search/
88	Disallow: /?q=user/password/
89	Disallow: /?q=user/register/
90	Disallow: /?q=user/login/
91	Disallow: /?q=user/logout/
92	Disallow: /?frontpage=true
93	Disallow: /?ref=[\w&='" _. ]+/
94
95	# wordpress
96
97	Disallow: /cgi-bin
98	Disallow: /wp-admin
99	Disallow: /wp-includes
100	Disallow: /wp-content/plugins
101	Disallow: /wp-content/cache
102	Disallow: /wp-content/themes
103	Disallow: /trackback
104	Disallow: /feed
105	Disallow: /comments
106	Disallow: /category//
107	Disallow: */trackback
108	Disallow: */feed
109	Disallow: */comments
110	Disallow: /?
111	Disallow: /*?
112	Allow: /wp-content/uploads
113
114	# Google Image
115	User-agent: Googlebot-Image
116	Allow: /*
117	Disallow: /cgi-bin
118	Disallow: /wp-admin
119	Disallow: /wp-includes
120	Disallow: /trackback
121	Disallow: /comments