RWE / robots.txt snapshot

← back to rwe.com · fetched 2026-06-20T01:10:30Z (15h ago) · HTTP 200 · 1304 bytes · sha256 6a84340a17f8c40d · raw

final URL: https://www.rwe.com/robots.txt

1	Sitemap: https://www.rwe.com/sitemap.xml
2
3	User-agent: AhrefsBot
4	Disallow: /
5
6	User-agent: MJ12bot
7	Disallow: /
8
9	User-agent: SemrushBot
10	Disallow: /
11
12	User-agent: YisouSpider
13	Disallow: /
14
15	User-agent: XoviBot
16	Disallow: /
17
18	User-agent: AdmantxBot
19	Disallow: /
20
21	User-agent: Proximic
22	Disallow: /
23
24	User-agent: WordPress
25	Disallow: /wp-admin/
26
27	User-agent: SISTRIX Crawler
28	Disallow: /
29
30	User-agent: DataForSeoBot
31	Disallow: /
32
33	User-Agent: thinkers-bot
34	Disallow: /
35
36	User-Agent: SiteimproveBot
37	Disallow: /
38
39	User-Agent: SiteimproveBot-Crawler
40	Disallow: /
41
42	User-agent: ALittle Client
43	Disallow:/
44
45	User-agent: *
46	# directory
47	Disallow: /App_Browsers/
48	Disallow: /App_Config/
49	Disallow: /App_Start/
50	Disallow: /Areas/
51	Disallow: /Artifacts/
52	Disallow: /bin/
53	Disallow: /Content/
54	Disallow: /layouts/
55	Disallow: /Properties/
56	Disallow: /Scripts/
57	Disallow: /sitecore/
58	Disallow: /sitecore modules/
59	Disallow: /sitecore_files/
60	Disallow: /temp/
61	Disallow: /upload/
62	Disallow: /Views/
63	Disallow: /xsl/
64	Disallow: /sitecore/Inhalt/
65	Disallow: /sitecore/Content/
66
67	# files
68	Disallow: /Default.aspx
69	Disallow: /default.css
70	Disallow: /default.htm.sitedown
71	Disallow: /default.js
72	Disallow: /Web.config.install.xdt
73	Disallow: /Web.config.uninstall.xdt
74	Disallow: /webedit.css
75	Disallow: /Global.asax
76	Disallow: /lb.txt
77	Disallow: /web.config
78
79
80	Allow: /
81