NeuralCrawl

University of Cape Town / robots.txt snapshot

← back to uct.ac.za · fetched 2026-06-26T14:15:22Z (4h ago) · HTTP 200 · 3792 bytes · sha256 c2a7d0a8154ed8db · raw

final URL: https://uct.ac.za/robots.txt

1# robots.txt
2
3User-agent: *
4# CSS, JS, Images
5Allow: /core/*.css$
6Allow: /core/*.css?
7Allow: /core/*.js$
8Allow: /core/*.js?
9Allow: /core/*.gif
10Allow: /core/*.jpg
11Allow: /core/*.jpeg
12Allow: /core/*.png
13Allow: /core/*.svg
14Allow: /profiles/*.css$
15Allow: /profiles/*.css?
16Allow: /profiles/*.js$
17Allow: /profiles/*.js?
18Allow: /profiles/*.gif
19Allow: /profiles/*.jpg
20Allow: /profiles/*.jpeg
21Allow: /profiles/*.png
22Allow: /profiles/*.svg
23# Directories
24Disallow: /core/
25Disallow: /profiles/
26# Files
27Disallow: /README.txt
28Disallow: /web.config
29# Paths (clean URLs)
30Disallow: /admin/
31Disallow: /comment/reply/
32Disallow: /filter/tips
33Disallow: /node/add/
34Disallow: /search
35Disallow: /search/
36Disallow: /*/search
37Disallow: /*/search/
38Disallow: /*/search$
39Disallow: /*/search?
40Disallow: /*?*search=
41Disallow: /user/register
42Disallow: /user/password
43Disallow: /user/login
44Disallow: /user/logout
45Disallow: /media/oembed
46Disallow: /*/media/oembed
47# Paths (no clean URLs)
48Disallow: /index.php/admin/
49Disallow: /index.php/comment/reply/
50Disallow: /index.php/filter/tips
51Disallow: /index.php/node/add/
52Disallow: /index.php/search/
53Disallow: /index.php/user/password
54Disallow: /index.php/user/register
55Disallow: /index.php/user/login
56Disallow: /index.php/user/logout
57Disallow: /index.php/media/oembed
58Disallow: /index.php/*/media/oembed
59
60User-agent: Googlebot
61Disallow: /private/
62Disallow: /secret.html
63Disallow: /tmp/
64Disallow: /cgi-bin/
65Disallow: /admin/
66Disallow: /search
67Disallow: /search/
68Disallow: /*/search
69Disallow: /*/search/
70Disallow: /*/search$
71Disallow: /*/search?
72Disallow: /*?*search=
73
74User-agent: Bingbot
75Disallow: /
76
77User-agent: Slurp
78Disallow: /private/
79Disallow: /secret.html
80Disallow: /tmp/
81Disallow: /cgi-bin/
82Disallow: /admin/
83Disallow: /search
84Disallow: /search/
85Disallow: /*/search
86Disallow: /*/search/
87Disallow: /*/search$
88Disallow: /*/search?
89Disallow: /*?*search=
90
91User-agent: DuckDuckBot
92Disallow: /private/
93Disallow: /secret.html
94Disallow: /tmp/
95Disallow: /cgi-bin/
96Disallow: /admin/
97Disallow: /search
98Disallow: /search/
99Disallow: /*/search
100Disallow: /*/search/
101Disallow: /*/search$
102Disallow: /*/search?
103Disallow: /*?*search=
104
105User-agent: Baiduspider
106Disallow: /private/
107Disallow: /secret.html
108Disallow: /tmp/
109Disallow: /cgi-bin/
110Disallow: /admin/
111Disallow: /search
112Disallow: /search/
113Disallow: /*/search
114Disallow: /*/search/
115Disallow: /*/search$
116Disallow: /*/search?
117Disallow: /*?*search=
118
119User-agent: Yandex
120Disallow: /
121
122User-agent: Sogou Spider
123Disallow: /private/
124Disallow: /secret.html
125Disallow: /tmp/
126Disallow: /cgi-bin/
127Disallow: /admin/
128Disallow: /search
129Disallow: /search/
130Disallow: /*/search
131Disallow: /*/search/
132Disallow: /*/search$
133Disallow: /*/search?
134Disallow: /*?*search=
135
136User-agent: MJ12bot
137Disallow: /private/
138Disallow: /secret.html
139Disallow: /tmp/
140Disallow: /cgi-bin/
141Disallow: /admin/
142Disallow: /search
143Disallow: /search/
144Disallow: /*/search
145Disallow: /*/search/
146Disallow: /*/search$
147Disallow: /*/search?
148Disallow: /*?*search=
149
150User-agent: AhrefsBot
151Disallow: /private/
152Disallow: /secret.html
153Disallow: /tmp/
154Disallow: /cgi-bin/
155Disallow: /admin/
156Disallow: /search
157Disallow: /search/
158Disallow: /*/search
159Disallow: /*/search/
160Disallow: /*/search$
161Disallow: /*/search?
162Disallow: /*?*search=
163
164User-agent: SemrushBot
165Disallow: /private/
166Disallow: /secret.html
167Disallow: /tmp/
168Disallow: /cgi-bin/
169Disallow: /admin/
170Disallow: /search
171Disallow: /search/
172Disallow: /*/search
173Disallow: /*/search/
174Disallow: /*/search$
175Disallow: /*/search?
176Disallow: /*?*search=
177
178User-agent: GPTBot
179Disallow: /
180
181User-agent: FriendlyCrawler
182Disallow: /
183
184User-agent: YandexBot
185Disallow: /
186
187User-agent: ByteDanceSpider
188Disallow: /
189
190User-agent: ByteDance
191Disallow: /
192
193User-agent: Bytespider
194Disallow: /
195
196User-agent: OAI-SearchBot
197Disallow: /
198
199# sitemap.xml
200Sitemap: https://uct.ac.za/sitemap.xml