Albemarle / robots.txt snapshot

← back to albemarle.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 3281 bytes · sha256 fe58ff45a5314ec9 · raw

final URL: https://www.albemarle.com/robots.txt

1	#
2	# robots.txt
3	#
4	# This file is to prevent the crawling and indexing of certain parts
5	# of your site by web crawlers and spiders run by sites like Yahoo!
6	# and Google. By telling these "robots" where not to go on your site,
7	# you save bandwidth and server resources.
8	#
9	# This file will be ignored unless it is at the root of your host:
10	# Used: http://example.com/robots.txt
11	# Ignored: http://example.com/site/robots.txt
12	#
13	# For more information about the robots.txt standard, see:
14	# http://www.robotstxt.org/robotstxt.html
15
16	User-agent: *
17	# CSS, JS, Images
18	Allow: /core/*.css$
19	Allow: /core/*.css?
20	Allow: /core/*.js$
21	Allow: /core/*.js?
22	Allow: /core/*.gif
23	Allow: /core/*.jpg
24	Allow: /core/*.jpeg
25	Allow: /core/*.png
26	Allow: /core/*.svg
27	Allow: /profiles/*.css$
28	Allow: /profiles/*.css?
29	Allow: /profiles/*.js$
30	Allow: /profiles/*.js?
31	Allow: /profiles/*.gif
32	Allow: /profiles/*.jpg
33	Allow: /profiles/*.jpeg
34	Allow: /profiles/*.png
35	Allow: /profiles/*.svg
36	# Directories
37	Disallow: /core/
38	Disallow: /profiles/
39	# Files
40	Disallow: /README.md
41	Disallow: /composer/Metapackage/README.txt
42	Disallow: /composer/Plugin/ProjectMessage/README.md
43	Disallow: /composer/Plugin/Scaffold/README.md
44	Disallow: /composer/Plugin/VendorHardening/README.txt
45	Disallow: /composer/Template/README.txt
46	Disallow: /modules/README.txt
47	Disallow: /sites/README.txt
48	Disallow: /themes/README.txt
49	Disallow: /web.config
50	# Paths (clean URLs)
51	Disallow: /admin/
52	Disallow: /comment/reply/
53	Disallow: /filter/tips
54	Disallow: /node/add/
55	Disallow: /search/
56	Disallow: /user/register
57	Disallow: /user/password
58	Disallow: /user/login
59	Disallow: /user/logout
60	Disallow: /media/oembed
61	Disallow: /*/media/oembed
62	# Paths (no clean URLs)
63	Disallow: /index.php/admin/
64	Disallow: /index.php/comment/reply/
65	Disallow: /index.php/filter/tips
66	Disallow: /index.php/node/add/
67	Disallow: /index.php/search/
68	Disallow: /index.php/user/password
69	Disallow: /index.php/user/register
70	Disallow: /index.php/user/login
71	Disallow: /index.php/user/logout
72	Disallow: /index.php/media/oembed
73	Disallow: /index.php/*/media/oembed
74
75	Disallow: /search/jobs/*
76	Disallow: /search/jobs?keyword=*
77	Disallow: //search/jobs/
78	Disallow: //search/jobs?keyword=
79	Disallow: /search/news/*
80	Disallow: /search/news?keyword=*
81	Disallow: //search/news/
82	Disallow: //search/news?keyword=
83	Disallow: /search/products
84	Disallow: /search/products?keyword=*
85	Disallow: /*/search/products
86	Disallow: //search/products?keyword=
87	Disallow: /search/site/*
88	Disallow: /search/site?keyword=*
89	Disallow: //search/site/
90	Disallow: //search/site?keyword=
91
92	Disallow: /taxonomy/term/*
93	Disallow: /user/login
94	Disallow: /user/login/*
95
96	Sitemap: https://www.albemarle.com/sitemap.xml
97
98	Disallow: /hu/es/*
99	Disallow: /hu/de/*
100	Disallow: /hu/zh-hans/*
101	Disallow: /hu/ja/*
102	Disallow: /hu/ko/*
103
104	Disallow: /au/es/*
105	Disallow: /au/de/*
106	Disallow: /au/hu/*
107	Disallow: /au/zh-hans/*
108	Disallow: /au/ja/*
109	Disallow: /au/ko/*
110
111	Disallow: /cl/de/*
112	Disallow: /cl/hu/*
113	Disallow: /cl/zh-hans/*
114	Disallow: /cl/ja/*
115	Disallow: /cl/ko/*
116
117	Disallow: /de/es/*
118	Disallow: /de/hu/*
119	Disallow: /de/zh-hans/*
120	Disallow: /de/ja/*
121	Disallow: /de/ko/*
122
123	Disallow: /cn/es/*
124	Disallow: /cn/de/*
125	Disallow: /cn/hu/*
126	Disallow: /cn/ja/*
127	Disallow: /cn/ko/*
128
129	Disallow: /us/es/*
130	Disallow: /us/de/*
131	Disallow: /us/hu/*
132	Disallow: /us/zh-hans/*
133	Disallow: /us/ja/*
134	Disallow: /us/ko/*