The Independent / robots.txt snapshot

← back to independent.co.uk · fetched 2026-07-17T12:07:56Z (20d ago) · HTTP 200 · 1305 bytes · sha256 b45123ef0320e9ca · raw

final URL: https://www.independent.co.uk/robots.txt

1
2
3
4	User-agent: *
5
6	# Files
7	Disallow: /CHANGELOG.txt
8	Disallow: /cron.php
9	Disallow: /_fs-ch-
10	Disallow: /INSTALL.mysql.txt
11	Disallow: /INSTALL.pgsql.txt
12	Disallow: /INSTALL.sqlite.txt
13	Disallow: /install.php
14	Disallow: /INSTALL.txt
15	Disallow: /LICENSE.txt
16	Disallow: /MAINTAINERS.txt
17	Disallow: /update.php
18	Disallow: /UPGRADE.txt
19	Disallow: /xmlrpc.php
20	Disallow: /img/placeholder.gif
21	# Paths (clean URLs)
22	Disallow: /admin/
23	Disallow: /comment/reply/
24	Disallow: /filter/tips/
25	Disallow: /node/add/
26	Disallow: /search/
27	Disallow: /user/register/
28	Disallow: /user/password/
29	Disallow: /user/login/
30	Disallow: /user/logout/
31	Disallow: /internal-api/
32	# Paths (no clean URLs)
33	Disallow: /?q=admin/
34	Disallow: /?q=comment/reply/
35	Disallow: /?q=filter/tips/
36	Disallow: /?q=node/add/
37	Disallow: /?q=search/
38	Disallow: /?q=user/password/
39	Disallow: /?q=user/register/
40	Disallow: /?q=user/login/
41	Disallow: /?q=user/logout/
42	Disallow: /pugpig/
43	Disallow: /videorequest
44	Disallow: /api/
45	Disallow: /api/html/
46	Disallow: /api/user/
47	Disallow: /71347885/
48	Disallow: /cb
49
50	# Ignore liveblog pagination and swipe tracking
51	Disallow: *itm_channel=native
52	Disallow: *?page=
53
54	# Ignore refresh URLs
55	Disallow: /*ILC-refresh
56
57	User-agent: Nutch
58	Disallow: /
59	Sitemap: https://www.independent.co.uk/sitemaps/googlenews
60	Sitemap: https://www.independent.co.uk/sitemap.xml
61