NeuralCrawl

South China Morning Post / robots.txt snapshot

← back to scmp.com · fetched 2026-08-05T06:56:07Z (22h ago) · HTTP 200 · 4536 bytes · sha256 2b63d896402c6467 · raw

final URL: https://www.scmp.com/robots.txt

1# robots.txt
2# This file is to prevent the crawling and indexing of certain parts
3# of your site by web crawlers and spiders run by sites like Yahoo!
4# and Google. By telling these "robots" where not to go on your site,
5# you save bandwidth and server resources.
6# This file will be ignored unless it is at the root of your host:
7# Used: http://example.com/robots.txt
8# Ignored: http://example.com/site/robots.txt
9# For more information about the robots.txt standard, see:
10# http://www.robotstxt.org/robotstxt.html
11# For syntax checking, see:
12# http://www.sxw.org.uk/computing/robots/check.html
13
14# AmazonAdBot
15User-agent: AmazonAdBot
16Allow: /
17
18# NewsNow
19User-agent: NewsNow
20Allow: /print/
21Allow: /?q=print/
22
23# GrapeShot
24User-agent: grapeshot
25Allow: /*/article/*$
26
27# Facebook
28User-agent: facebookexternalhit
29Allow: /
30
31User-agent: *
32Crawl-delay: 10
33
34# Ads
35Disallow: /*?*campaign=*
36Disallow: /*?*cid=*
37Disallow: /*?*nochartbeat=*
38Disallow: /*?*noga=*
39Disallow: /*?*nograpeshot=*
40Disallow: /*?*nogtm=*
41Disallow: /*?*noixwrapper=*
42Disallow: /*?*nomoat=*
43Disallow: /*?*nomoatyi=*
44Disallow: /*?*showonlyads=*
45
46# Bot score and country
47Disallow: /navigator-info
48
49# Path
50Disallow: /.well-known/*
51Disallow: /*/firebase-messaging-sw.js
52Disallow: /google97d8d43559c9b155.html
53Disallow: /healthz
54Disallow: /login
55Disallow: /login/facebook
56Disallow: /login/facebook/*
57Disallow: /logout
58Disallow: /register
59Disallow: /signin
60Disallow: /styleguide/*
61
62# Directories
63Disallow: /includes/
64Disallow: /misc/
65Disallow: /modules/
66Disallow: /profiles/
67Disallow: /public/
68Disallow: /scripts/
69Disallow: /static/
70Disallow: /themes/
71
72# Files
73Disallow: /_next/data/*.json
74Disallow: /CHANGELOG.txt
75Disallow: /cron.php
76Disallow: /INSTALL.mysql.txt
77Disallow: /INSTALL.pgsql.txt
78Disallow: /install.php
79Disallow: /INSTALL.sqlite.txt
80Disallow: /INSTALL.txt
81Disallow: /LICENSE.txt
82Disallow: /MAINTAINERS.txt
83Disallow: /sites/default/files/*.doc
84Disallow: /sites/default/files/*.docx
85Disallow: /sites/default/files/*.pdf
86Disallow: /sites/default/files/*.swf
87Disallow: /sites/default/files/styles/*.jpg
88Disallow: /update.php
89Disallow: /UPGRADE.txt
90Disallow: /xmlrpc.php
91
92# Paths (clean URLs)
93Disallow: *?destination=*
94Disallow: *Article_type=*
95Disallow: *field_article*
96Disallow: /admin/
97Disallow: /ajax
98Disallow: /ajax_comments/
99Disallow: /ajax/*
100Disallow: /comment/reply/
101Disallow: /epaper
102Disallow: /epaper/*
103Disallow: /facebook-instant-articles/feed/*
104Disallow: /filter/tips/
105Disallow: /label/
106Disallow: /node/*/edit
107Disallow: /node/*/nodequeue
108Disallow: /node/add/
109Disallow: /scmp_comments/
110Disallow: /story/style/*
111Disallow: /user/login/
112Disallow: /user/logout/
113Disallow: /user/password/
114Disallow: /user/register/
115
116# Paths (no clean URLs)
117Disallow: /?q=admin/
118Disallow: /?q=comment/reply/
119Disallow: /?q=epaper
120Disallow: /?q=epaper/*
121Disallow: /?q=facebook-instant-articles/feed/*
122Disallow: /?q=filter/tips/
123Disallow: /?q=node/*/edit
124Disallow: /?q=node/*/nodequeue
125Disallow: /?q=node/add/
126Disallow: /?q=print/
127Disallow: /?q=user/login/
128Disallow: /?q=user/logout/
129Disallow: /?q=user/password/
130Disallow: /?q=user/register/
131Disallow: /*/antiSpam$
132Disallow: /*/beaconBase$
133Disallow: /*/beforeUnload$
134Disallow: /*/client$
135Disallow: /*/compose$
136Disallow: /*/cookie$
137Disallow: /*/etag$
138Disallow: /*/goldlog$
139Disallow: /*/iframe$
140Disallow: /*/initLoad$
141Disallow: /*/lib_b/*$
142Disallow: /*/log$
143Disallow: /*/logSend$
144Disallow: /*/makeid$
145Disallow: /*/metaInfo$
146Disallow: /*/misc$
147Disallow: /*/nameStorage$
148Disallow: /*/personality/index$
149Disallow: /*/pvData$
150Disallow: /*/pvid$
151Disallow: /*/referrer$
152Disallow: /*/sendpv$
153Disallow: /*/spm$
154Disallow: /*/spmact$
155Disallow: /*/spmException$
156Disallow: /*/spmMonitor$
157Disallow: /*/util$
158Disallow: /*/windvane$
159Disallow: /print/
160
161# Sitemap
162Sitemap: https://www.scmp.com/sitemap/sitemap.xml
163Sitemap: https://www.scmp.com/sitemap/archives-0.xml
164
165# Temp removal
166Disallow: /presented/better-life/career-education/topics/edtalk-harbour-school/article/3199785/rethinking-5-educational-myths-through-experiential-learning-approach
167Disallow: /print/presented/better-life/career-education/topics/edtalk-harbour-school/article/3199785/rethinking-5-educational-myths-through-experiential-learning-approach
168Disallow: /native/business/banking-finance/topics/wealth-across-borders/article/3357488/retired-businessman-and-lifelong-sailor-trust-anchors-process-building-wealth
169Disallow: /print/native/business/banking-finance/topics/wealth-across-borders/article/3357488/retired-businessman-and-lifelong-sailor-trust-anchors-process-building-wealth