NeuralCrawl

South China Morning Post / robots.txt

← back to scmp.com · change detected 7d ago (2026-07-28T10:03:18Z) · modified +122 −154

What changed

Diff old (2026-07-15T11:19:22Z) → new (2026-07-28T10:03:18Z)

@@ -1,164 +1,19 @@
1- #
21  # robots.txt
3- #
42  # This file is to prevent the crawling and indexing of certain parts
53  # of your site by web crawlers and spiders run by sites like Yahoo!
64  # and Google. By telling these "robots" where not to go on your site,
75  # you save bandwidth and server resources.
8- #
96  # This file will be ignored unless it is at the root of your host:
107  # Used: http://example.com/robots.txt
118  # Ignored: http://example.com/site/robots.txt
12- #
139  # For more information about the robots.txt standard, see:
1410  # http://www.robotstxt.org/robotstxt.html
15- #
1611  # For syntax checking, see:
1712  # http://www.sxw.org.uk/computing/robots/check.html
1813 
1914  # AmazonAdBot
2015  User-agent: AmazonAdBot
2116  Allow: /
22-
23- User-agent: *
24- Crawl-delay: 10
25-
26- # PWA
27-
28- # Directories
29- Disallow: /public/
30- Disallow: /static/
31- # Path
32- Disallow: /login
33- Disallow: /signin
34- Disallow: /register
35- Disallow: /logout
36- Disallow: /login/facebook
37- Disallow: /login/facebook/*
38- Disallow: /styleguide/*
39- Disallow: /healthz
40- Disallow: /.well-known/*
41- Disallow: /*/firebase-messaging-sw.js
42- Disallow: /google97d8d43559c9b155.html
43- Allow: /.well-known/amphtml/apikey.pub
44-
45- # CSS, JS, Image
46- Allow: /static/*.css$
47- Allow: /static/*.css?
48- Allow: /static/*.js$
49- Allow: /static/*.js?
50- Allow: /static/*.gif
51- Allow: /static/*.jpg
52- Allow: /static/*.jpeg
53- Allow: /static/*.png
54- Allow: /public/*.css$
55- Allow: /public/*.css?
56- Allow: /public/*.js$
57- Allow: /public/*.js?
58- Allow: /public/*.gif
59- Allow: /public/*.jpg
60- Allow: /public/*.jpeg
61- Allow: /public/*.png
62-
63- # Directories
64- Disallow: /includes/
65- Disallow: /misc/
66- Disallow: /modules/
67- Disallow: /profiles/
68- Disallow: /scripts/
69- Disallow: /themes/
70- # Files
71- Disallow: /CHANGELOG.txt
72- Disallow: /cron.php
73- Disallow: /INSTALL.mysql.txt
74- Disallow: /INSTALL.pgsql.txt
75- Disallow: /INSTALL.sqlite.txt
76- Disallow: /install.php
77- Disallow: /INSTALL.txt
78- Disallow: /LICENSE.txt
79- Disallow: /MAINTAINERS.txt
80- Disallow: /update.php
81- Disallow: /UPGRADE.txt
82- Disallow: /xmlrpc.php
83- Disallow: /sites/default/files/*.pdf
84- Disallow: /sites/default/files/*.doc
85- Disallow: /sites/default/files/*.docx
86- Disallow: /sites/default/files/*.swf
87- Disallow: /sites/default/files/styles/*.jpg
88- Disallow: /_next/data/*.json
89-
90- # Paths (clean URLs)
91- Disallow: /admin/
92- Disallow: /comment/reply/
93- Disallow: /filter/tips/
94- Disallow: /node/add/
95- Disallow: /user/register/
96- Disallow: /user/password/
97- Disallow: /user/login/
98- Disallow: /user/logout/
99- Disallow: *?destination=*
100- Disallow: /ajax_comments/
101- Disallow: /scmp_comments/
102- Disallow: *Article_type=*
103- Disallow: *field_article*
104- Disallow: /label/
105- Disallow: /node/*/nodequeue
106- Disallow: /node/*/edit
107- Disallow: /ajax
108- Disallow: /ajax/*
109- Disallow: /facebook-instant-articles/feed/*
110- Disallow: /epaper
111- Disallow: /epaper/*
112- Disallow: /story/style/*
113-
114- # Paths (no clean URLs)
115- Disallow: /?q=admin/
116- Disallow: /?q=comment/reply/
117- Disallow: /?q=filter/tips/
118- Disallow: /?q=node/add/
119- Disallow: /?q=user/password/
120- Disallow: /?q=user/register/
121- Disallow: /?q=user/login/
122- Disallow: /?q=user/logout/
123- Disallow: /?q=node/*/edit
124- Disallow: /?q=node/*/nodequeue
125- Disallow: /?q=epaper
126- Disallow: /?q=epaper/*
127- Disallow: /?q=facebook-instant-articles/feed/*
128-
129- Disallow: /*/logSend$
130- Disallow: /*/spmException$
131- Disallow: /*/spmact$
132- Disallow: /*/antiSpam$
133- Disallow: /*/nameStorage$
134- Disallow: /*/spmMonitor$
135- Disallow: /*/pvData$
136- Disallow: /*/goldlog$
137- Disallow: /*/initLoad$
138- Disallow: /*/beforeUnload$
139- Disallow: /*/util$
140- Disallow: /*/metaInfo$
141- Disallow: /*/beaconBase$
142- Disallow: /*/spm$
143- Disallow: /*/makeid$
144- Disallow: /*/referrer$
145- Disallow: /*/pvid$
146- Disallow: /*/etag$
147- Disallow: /*/iframe$
148- Disallow: /*/client$
149- Disallow: /*/windvane$
150- Disallow: /*/cookie$
151- Disallow: /*/sendpv$
152- Disallow: /*/personality/index$
153- Disallow: /*/misc$
154- Disallow: /*/client$
155- Disallow: /*/log$
156- Disallow: /*/compose$
157- Disallow: /*/lib_b/*$
158- Disallow: /print/
159- Disallow: /?q=print/
160- Disallow: /search?q=*
161- Disallow: /search/*
16217 
16318  # NewsNow
16419  User-agent: NewsNow
@@ -169,26 +24,139 @@
16924  User-agent: grapeshot
17025  Allow: /*/article/*$
17126 
27+ User-agent: *
28+ Crawl-delay: 10
29+
17230  # Ads
31+ Disallow: /*?*campaign=*
17332  Disallow: /*?*cid=*
174- Disallow: /*?*showonlyads=*
175- Disallow: /*?*nograpeshot=*
176- Disallow: /*?*noixwrapper=*
177- Disallow: /*?*nogtm=*
17833  Disallow: /*?*nochartbeat=*
17934  Disallow: /*?*noga=*
35+ Disallow: /*?*nograpeshot=*
36+ Disallow: /*?*nogtm=*
37+ Disallow: /*?*noixwrapper=*
38+ Disallow: /*?*nomoat=*
18039  Disallow: /*?*nomoatyi=*
181- Disallow: /*?*nomoat=*
40+ Disallow: /*?*showonlyads=*
18241 
18342  # Bot score and country
18443  Disallow: /navigator-info
18544 
186- # Disallow article with query params and everything with campaign for google bot
187- User-agent: Googlebot
188- Disallow: /*?*campaign=*
45+ # Path
46+ Disallow: /.well-known/*
47+ Disallow: /*/firebase-messaging-sw.js
48+ Disallow: /google97d8d43559c9b155.html
49+ Disallow: /healthz
50+ Disallow: /login
51+ Disallow: /login/facebook
52+ Disallow: /login/facebook/*
53+ Disallow: /logout
54+ Disallow: /register
55+ Disallow: /signin
56+ Disallow: /styleguide/*
57+
58+ # Directories
59+ Disallow: /includes/
60+ Disallow: /misc/
61+ Disallow: /modules/
62+ Disallow: /profiles/
63+ Disallow: /public/
64+ Disallow: /scripts/
65+ Disallow: /static/
66+ Disallow: /themes/
67+
68+ # Files
69+ Disallow: /_next/data/*.json
70+ Disallow: /CHANGELOG.txt
71+ Disallow: /cron.php
72+ Disallow: /INSTALL.mysql.txt
73+ Disallow: /INSTALL.pgsql.txt
74+ Disallow: /install.php
75+ Disallow: /INSTALL.sqlite.txt
76+ Disallow: /INSTALL.txt
77+ Disallow: /LICENSE.txt
78+ Disallow: /MAINTAINERS.txt
79+ Disallow: /sites/default/files/*.doc
80+ Disallow: /sites/default/files/*.docx
81+ Disallow: /sites/default/files/*.pdf
82+ Disallow: /sites/default/files/*.swf
83+ Disallow: /sites/default/files/styles/*.jpg
84+ Disallow: /update.php
85+ Disallow: /UPGRADE.txt
86+ Disallow: /xmlrpc.php
87+
88+ # Paths (clean URLs)
89+ Disallow: *?destination=*
90+ Disallow: *Article_type=*
91+ Disallow: *field_article*
92+ Disallow: /admin/
93+ Disallow: /ajax
94+ Disallow: /ajax_comments/
95+ Disallow: /ajax/*
96+ Disallow: /comment/reply/
97+ Disallow: /epaper
98+ Disallow: /epaper/*
99+ Disallow: /facebook-instant-articles/feed/*
100+ Disallow: /filter/tips/
101+ Disallow: /label/
102+ Disallow: /node/*/edit
103+ Disallow: /node/*/nodequeue
104+ Disallow: /node/add/
105+ Disallow: /scmp_comments/
106+ Disallow: /story/style/*
107+ Disallow: /user/login/
108+ Disallow: /user/logout/
109+ Disallow: /user/password/
110+ Disallow: /user/register/
111+
112+ # Paths (no clean URLs)
113+ Disallow: /?q=admin/
114+ Disallow: /?q=comment/reply/
115+ Disallow: /?q=epaper
116+ Disallow: /?q=epaper/*
117+ Disallow: /?q=facebook-instant-articles/feed/*
118+ Disallow: /?q=filter/tips/
119+ Disallow: /?q=node/*/edit
120+ Disallow: /?q=node/*/nodequeue
121+ Disallow: /?q=node/add/
122+ Disallow: /?q=print/
123+ Disallow: /?q=user/login/
124+ Disallow: /?q=user/logout/
125+ Disallow: /?q=user/password/
126+ Disallow: /?q=user/register/
127+ Disallow: /*/antiSpam$
128+ Disallow: /*/beaconBase$
129+ Disallow: /*/beforeUnload$
130+ Disallow: /*/client$
131+ Disallow: /*/compose$
132+ Disallow: /*/cookie$
133+ Disallow: /*/etag$
134+ Disallow: /*/goldlog$
135+ Disallow: /*/iframe$
136+ Disallow: /*/initLoad$
137+ Disallow: /*/lib_b/*$
138+ Disallow: /*/log$
139+ Disallow: /*/logSend$
140+ Disallow: /*/makeid$
141+ Disallow: /*/metaInfo$
142+ Disallow: /*/misc$
143+ Disallow: /*/nameStorage$
144+ Disallow: /*/personality/index$
145+ Disallow: /*/pvData$
146+ Disallow: /*/pvid$
147+ Disallow: /*/referrer$
148+ Disallow: /*/sendpv$
149+ Disallow: /*/spm$
150+ Disallow: /*/spmact$
151+ Disallow: /*/spmException$
152+ Disallow: /*/spmMonitor$
153+ Disallow: /*/util$
154+ Disallow: /*/windvane$
155+ Disallow: /print/
156+ Disallow: /search?q=*
157+ Disallow: /search/*
189158 
190159  # Sitemap
191-
192160  Sitemap: https://www.scmp.com/sitemap/sitemap.xml
193161  Sitemap: https://www.scmp.com/sitemap/archives-0.xml
194162