NeuralCrawl

University of Cape Town / robots.txt snapshot

← back to uct.ac.za · fetched 2026-07-01T12:24:12Z (1mo ago) · HTTP 200 · 4246 bytes · sha256 708c99271cbdac66 · raw

final URL: https://uct.ac.za/robots.txt

1# robots.txt
2
3User-agent: *
4# CSS, JS, Images
5Allow: /core/*.css$
6Allow: /core/*.css?
7Allow: /core/*.js$
8Allow: /core/*.js?
9Allow: /core/*.gif
10Allow: /core/*.jpg
11Allow: /core/*.jpeg
12Allow: /core/*.png
13Allow: /core/*.svg
14Allow: /profiles/*.css$
15Allow: /profiles/*.css?
16Allow: /profiles/*.js$
17Allow: /profiles/*.js?
18Allow: /profiles/*.gif
19Allow: /profiles/*.jpg
20Allow: /profiles/*.jpeg
21Allow: /profiles/*.png
22Allow: /profiles/*.svg
23# Directories
24Disallow: /core/
25Disallow: /profiles/
26# Files
27Disallow: /README.txt
28Disallow: /web.config
29# Paths (clean URLs)
30Disallow: /admin/
31Disallow: /comment/reply/
32Disallow: /filter/tips
33Disallow: /node/add/
34Disallow: /search
35Disallow: /search/
36Disallow: /*/search
37Disallow: /*/search/
38Disallow: /*/search$
39Disallow: /*/search?
40Disallow: /*?*search=
41Disallow: /user/register
42Disallow: /user/password
43Disallow: /user/login
44Disallow: /user/logout
45Disallow: /media/oembed
46Disallow: /*/media/oembed
47# Paths (no clean URLs)
48Disallow: /index.php/admin/
49Disallow: /index.php/comment/reply/
50Disallow: /index.php/filter/tips
51Disallow: /index.php/node/add/
52Disallow: /index.php/search/
53Disallow: /index.php/user/password
54Disallow: /index.php/user/register
55Disallow: /index.php/user/login
56Disallow: /index.php/user/logout
57Disallow: /index.php/media/oembed
58Disallow: /index.php/*/media/oembed
59
60User-agent: Googlebot
61Disallow: /private/
62Disallow: /secret.html
63Disallow: /tmp/
64Disallow: /cgi-bin/
65Disallow: /admin/
66Disallow: /search
67Disallow: /search/
68Disallow: /*/search
69Disallow: /*/search/
70Disallow: /*/search$
71Disallow: /*/search?
72Disallow: /*?*search=
73
74User-agent: Bingbot
75Disallow: /private/
76Disallow: /secret.html
77Disallow: /tmp/
78Disallow: /cgi-bin/
79Disallow: /admin/
80Disallow: /search
81Disallow: /search/
82Disallow: /*/search
83Disallow: /*/search/
84Disallow: /*/search$
85Disallow: /*/search?
86Disallow: /*?*search=
87
88User-agent: Slurp
89Disallow: /private/
90Disallow: /secret.html
91Disallow: /tmp/
92Disallow: /cgi-bin/
93Disallow: /admin/
94Disallow: /search
95Disallow: /search/
96Disallow: /*/search
97Disallow: /*/search/
98Disallow: /*/search$
99Disallow: /*/search?
100Disallow: /*?*search=
101
102User-agent: DuckDuckBot
103Disallow: /private/
104Disallow: /secret.html
105Disallow: /tmp/
106Disallow: /cgi-bin/
107Disallow: /admin/
108Disallow: /search
109Disallow: /search/
110Disallow: /*/search
111Disallow: /*/search/
112Disallow: /*/search$
113Disallow: /*/search?
114Disallow: /*?*search=
115
116User-agent: Baiduspider
117Disallow: /private/
118Disallow: /secret.html
119Disallow: /tmp/
120Disallow: /cgi-bin/
121Disallow: /admin/
122Disallow: /search
123Disallow: /search/
124Disallow: /*/search
125Disallow: /*/search/
126Disallow: /*/search$
127Disallow: /*/search?
128Disallow: /*?*search=
129
130User-agent: Yandex
131Disallow: /
132
133User-agent: Sogou Spider
134Disallow: /private/
135Disallow: /secret.html
136Disallow: /tmp/
137Disallow: /cgi-bin/
138Disallow: /admin/
139Disallow: /search
140Disallow: /search/
141Disallow: /*/search
142Disallow: /*/search/
143Disallow: /*/search$
144Disallow: /*/search?
145Disallow: /*?*search=
146
147User-agent: MJ12bot
148Disallow: /private/
149Disallow: /secret.html
150Disallow: /tmp/
151Disallow: /cgi-bin/
152Disallow: /admin/
153Disallow: /search
154Disallow: /search/
155Disallow: /*/search
156Disallow: /*/search/
157Disallow: /*/search$
158Disallow: /*/search?
159Disallow: /*?*search=
160
161User-agent: AhrefsBot
162Disallow: /private/
163Disallow: /secret.html
164Disallow: /tmp/
165Disallow: /cgi-bin/
166Disallow: /admin/
167Disallow: /search
168Disallow: /search/
169Disallow: /*/search
170Disallow: /*/search/
171Disallow: /*/search$
172Disallow: /*/search?
173Disallow: /*?*search=
174
175User-agent: SemrushBot
176Disallow: /private/
177Disallow: /secret.html
178Disallow: /tmp/
179Disallow: /cgi-bin/
180Disallow: /admin/
181Disallow: /search
182Disallow: /search/
183Disallow: /*/search
184Disallow: /*/search/
185Disallow: /*/search$
186Disallow: /*/search?
187Disallow: /*?*search=
188
189User-agent: GPTBot
190Disallow: /
191
192User-agent: FriendlyCrawler
193Disallow: /
194
195User-agent: YandexBot
196Disallow: /
197
198User-agent: ByteDanceSpider
199Disallow: /
200
201User-agent: ByteDance
202Disallow: /
203
204User-agent: Bytespider
205Disallow: /
206
207User-agent: OAI-SearchBot
208Disallow: /private/
209Disallow: /secret.html
210Disallow: /tmp/
211Disallow: /cgi-bin/
212Disallow: /admin/
213Disallow: /search
214Disallow: /search/
215Disallow: /*/search
216Disallow: /*/search/
217Disallow: /*/search$
218Disallow: /*/search?
219Disallow: /*?*search=
220
221# sitemap.xml
222Sitemap: https://uct.ac.za/sitemap.xml