NeuralCrawl

South China Morning Post / robots.txt snapshot

← back to scmp.com · fetched 2026-07-28T10:03:18Z (8d ago) · HTTP 200 · 4525 bytes · sha256 ad91e3fd09fae860 · raw

final URL: https://www.scmp.com/robots.txt

1# robots.txt
2# This file is to prevent the crawling and indexing of certain parts
3# of your site by web crawlers and spiders run by sites like Yahoo!
4# and Google. By telling these "robots" where not to go on your site,
5# you save bandwidth and server resources.
6# This file will be ignored unless it is at the root of your host:
7# Used: http://example.com/robots.txt
8# Ignored: http://example.com/site/robots.txt
9# For more information about the robots.txt standard, see:
10# http://www.robotstxt.org/robotstxt.html
11# For syntax checking, see:
12# http://www.sxw.org.uk/computing/robots/check.html
13
14# AmazonAdBot
15User-agent: AmazonAdBot
16Allow: /
17
18# NewsNow
19User-agent: NewsNow
20Allow: /print/
21Allow: /?q=print/
22
23# GrapeShot
24User-agent: grapeshot
25Allow: /*/article/*$
26
27User-agent: *
28Crawl-delay: 10
29
30# Ads
31Disallow: /*?*campaign=*
32Disallow: /*?*cid=*
33Disallow: /*?*nochartbeat=*
34Disallow: /*?*noga=*
35Disallow: /*?*nograpeshot=*
36Disallow: /*?*nogtm=*
37Disallow: /*?*noixwrapper=*
38Disallow: /*?*nomoat=*
39Disallow: /*?*nomoatyi=*
40Disallow: /*?*showonlyads=*
41
42# Bot score and country
43Disallow: /navigator-info
44
45# Path
46Disallow: /.well-known/*
47Disallow: /*/firebase-messaging-sw.js
48Disallow: /google97d8d43559c9b155.html
49Disallow: /healthz
50Disallow: /login
51Disallow: /login/facebook
52Disallow: /login/facebook/*
53Disallow: /logout
54Disallow: /register
55Disallow: /signin
56Disallow: /styleguide/*
57
58# Directories
59Disallow: /includes/
60Disallow: /misc/
61Disallow: /modules/
62Disallow: /profiles/
63Disallow: /public/
64Disallow: /scripts/
65Disallow: /static/
66Disallow: /themes/
67
68# Files
69Disallow: /_next/data/*.json
70Disallow: /CHANGELOG.txt
71Disallow: /cron.php
72Disallow: /INSTALL.mysql.txt
73Disallow: /INSTALL.pgsql.txt
74Disallow: /install.php
75Disallow: /INSTALL.sqlite.txt
76Disallow: /INSTALL.txt
77Disallow: /LICENSE.txt
78Disallow: /MAINTAINERS.txt
79Disallow: /sites/default/files/*.doc
80Disallow: /sites/default/files/*.docx
81Disallow: /sites/default/files/*.pdf
82Disallow: /sites/default/files/*.swf
83Disallow: /sites/default/files/styles/*.jpg
84Disallow: /update.php
85Disallow: /UPGRADE.txt
86Disallow: /xmlrpc.php
87
88# Paths (clean URLs)
89Disallow: *?destination=*
90Disallow: *Article_type=*
91Disallow: *field_article*
92Disallow: /admin/
93Disallow: /ajax
94Disallow: /ajax_comments/
95Disallow: /ajax/*
96Disallow: /comment/reply/
97Disallow: /epaper
98Disallow: /epaper/*
99Disallow: /facebook-instant-articles/feed/*
100Disallow: /filter/tips/
101Disallow: /label/
102Disallow: /node/*/edit
103Disallow: /node/*/nodequeue
104Disallow: /node/add/
105Disallow: /scmp_comments/
106Disallow: /story/style/*
107Disallow: /user/login/
108Disallow: /user/logout/
109Disallow: /user/password/
110Disallow: /user/register/
111
112# Paths (no clean URLs)
113Disallow: /?q=admin/
114Disallow: /?q=comment/reply/
115Disallow: /?q=epaper
116Disallow: /?q=epaper/*
117Disallow: /?q=facebook-instant-articles/feed/*
118Disallow: /?q=filter/tips/
119Disallow: /?q=node/*/edit
120Disallow: /?q=node/*/nodequeue
121Disallow: /?q=node/add/
122Disallow: /?q=print/
123Disallow: /?q=user/login/
124Disallow: /?q=user/logout/
125Disallow: /?q=user/password/
126Disallow: /?q=user/register/
127Disallow: /*/antiSpam$
128Disallow: /*/beaconBase$
129Disallow: /*/beforeUnload$
130Disallow: /*/client$
131Disallow: /*/compose$
132Disallow: /*/cookie$
133Disallow: /*/etag$
134Disallow: /*/goldlog$
135Disallow: /*/iframe$
136Disallow: /*/initLoad$
137Disallow: /*/lib_b/*$
138Disallow: /*/log$
139Disallow: /*/logSend$
140Disallow: /*/makeid$
141Disallow: /*/metaInfo$
142Disallow: /*/misc$
143Disallow: /*/nameStorage$
144Disallow: /*/personality/index$
145Disallow: /*/pvData$
146Disallow: /*/pvid$
147Disallow: /*/referrer$
148Disallow: /*/sendpv$
149Disallow: /*/spm$
150Disallow: /*/spmact$
151Disallow: /*/spmException$
152Disallow: /*/spmMonitor$
153Disallow: /*/util$
154Disallow: /*/windvane$
155Disallow: /print/
156Disallow: /search?q=*
157Disallow: /search/*
158
159# Sitemap
160Sitemap: https://www.scmp.com/sitemap/sitemap.xml
161Sitemap: https://www.scmp.com/sitemap/archives-0.xml
162
163# Temp removal
164Disallow: /presented/better-life/career-education/topics/edtalk-harbour-school/article/3199785/rethinking-5-educational-myths-through-experiential-learning-approach
165Disallow: /print/presented/better-life/career-education/topics/edtalk-harbour-school/article/3199785/rethinking-5-educational-myths-through-experiential-learning-approach
166Disallow: /native/business/banking-finance/topics/wealth-across-borders/article/3357488/retired-businessman-and-lifelong-sailor-trust-anchors-process-building-wealth
167Disallow: /print/native/business/banking-finance/topics/wealth-across-borders/article/3357488/retired-businessman-and-lifelong-sailor-trust-anchors-process-building-wealth