NeuralCrawl

Coursera / robots.txt snapshot

← back to coursera.org · fetched 2026-09-11T00:58:05Z (8d ago) · HTTP 200 · 7533 bytes · sha256 aa59d410957cda64 · raw

final URL: https://www.coursera.org/robots.txt

1User-agent: *
2Allow: /api/utilities/v1/imageproxy
3Allow: /llms.txt
4Disallow: /maestro/api/
5Disallow: /api/
6Disallow: /maestro/
7Disallow: /ui/
8Disallow: /signature/voucher/
9Disallow: /account/
10Disallow: /acclaimbadge/
11Disallow: /voucher/
12Disallow: /search
13Disallow: /ent-website/
14Disallow: /learn-perf/
15Disallow: /specializations-perf/
16Disallow: /professional-certificates-perf/
17Disallow: /learn-noperf/
18Disallow: /specializations-noperf/
19Disallow: /professional-certificates-noperf/
20Disallow: /business/xmlrpc.php
21Disallow: /business/wp-content/uploads/ 
22Disallow: /business/search
23Disallow: /business/teams/search
24Disallow: /organizations/
25Disallow: /programs/
26
27# Block all /courses pagination
28Disallow: /courses?page=
29Disallow: /courses?*&page=
30
31# Allow pagination pages 1-5 when page is first
32Allow: /courses?page=1$
33Allow: /courses?page=1&
34Allow: /courses?page=2$
35Allow: /courses?page=2&
36Allow: /courses?page=3$
37Allow: /courses?page=3&
38Allow: /courses?page=4$
39Allow: /courses?page=4&
40Allow: /courses?page=5$
41Allow: /courses?page=5&
42
43# Allow pagination pages 1-5 when page appears after another parameter
44Allow: /courses?*&page=1$
45Allow: /courses?*&page=1&
46Allow: /courses?*&page=2$
47Allow: /courses?*&page=2&
48Allow: /courses?*&page=3$
49Allow: /courses?*&page=3&
50Allow: /courses?*&page=4$
51Allow: /courses?*&page=4&
52Allow: /courses?*&page=5$
53Allow: /courses?*&page=5&
54
55Sitemap: https://www.coursera.org/sitemap.xml
56
57
58User-agent: CCBot
59Disallow: /lecture/
60Disallow: /courses?page=
61Disallow: /courses?*&page=
62Allow: /courses?page=1$
63Allow: /courses?page=1&
64Allow: /courses?page=2$
65Allow: /courses?page=2&
66Allow: /courses?page=3$
67Allow: /courses?page=3&
68Allow: /courses?page=4$
69Allow: /courses?page=4&
70Allow: /courses?page=5$
71Allow: /courses?page=5&
72Allow: /courses?*&page=1$
73Allow: /courses?*&page=1&
74Allow: /courses?*&page=2$
75Allow: /courses?*&page=2&
76Allow: /courses?*&page=3$
77Allow: /courses?*&page=3&
78Allow: /courses?*&page=4$
79Allow: /courses?*&page=4&
80Allow: /courses?*&page=5$
81Allow: /courses?*&page=5&
82
83
84User-agent: GPTBot
85Disallow: /lecture/
86Disallow: /courses?page=
87Disallow: /courses?*&page=
88Allow: /courses?page=1$
89Allow: /courses?page=1&
90Allow: /courses?page=2$
91Allow: /courses?page=2&
92Allow: /courses?page=3$
93Allow: /courses?page=3&
94Allow: /courses?page=4$
95Allow: /courses?page=4&
96Allow: /courses?page=5$
97Allow: /courses?page=5&
98Allow: /courses?*&page=1$
99Allow: /courses?*&page=1&
100Allow: /courses?*&page=2$
101Allow: /courses?*&page=2&
102Allow: /courses?*&page=3$
103Allow: /courses?*&page=3&
104Allow: /courses?*&page=4$
105Allow: /courses?*&page=4&
106Allow: /courses?*&page=5$
107Allow: /courses?*&page=5&
108
109
110User-agent: ClaudeBot
111Disallow: /lecture/
112Disallow: /courses?page=
113Disallow: /courses?*&page=
114Allow: /courses?page=1$
115Allow: /courses?page=1&
116Allow: /courses?page=2$
117Allow: /courses?page=2&
118Allow: /courses?page=3$
119Allow: /courses?page=3&
120Allow: /courses?page=4$
121Allow: /courses?page=4&
122Allow: /courses?page=5$
123Allow: /courses?page=5&
124Allow: /courses?*&page=1$
125Allow: /courses?*&page=1&
126Allow: /courses?*&page=2$
127Allow: /courses?*&page=2&
128Allow: /courses?*&page=3$
129Allow: /courses?*&page=3&
130Allow: /courses?*&page=4$
131Allow: /courses?*&page=4&
132Allow: /courses?*&page=5$
133Allow: /courses?*&page=5&
134
135
136User-agent: MistralAI-User
137Disallow: /lecture/
138Disallow: /courses?page=
139Disallow: /courses?*&page=
140Allow: /courses?page=1$
141Allow: /courses?page=1&
142Allow: /courses?page=2$
143Allow: /courses?page=2&
144Allow: /courses?page=3$
145Allow: /courses?page=3&
146Allow: /courses?page=4$
147Allow: /courses?page=4&
148Allow: /courses?page=5$
149Allow: /courses?page=5&
150Allow: /courses?*&page=1$
151Allow: /courses?*&page=1&
152Allow: /courses?*&page=2$
153Allow: /courses?*&page=2&
154Allow: /courses?*&page=3$
155Allow: /courses?*&page=3&
156Allow: /courses?*&page=4$
157Allow: /courses?*&page=4&
158Allow: /courses?*&page=5$
159Allow: /courses?*&page=5&
160
161
162User-agent: Google-Extended
163Disallow: /lecture/
164Disallow: /courses?page=
165Disallow: /courses?*&page=
166Allow: /courses?page=1$
167Allow: /courses?page=1&
168Allow: /courses?page=2$
169Allow: /courses?page=2&
170Allow: /courses?page=3$
171Allow: /courses?page=3&
172Allow: /courses?page=4$
173Allow: /courses?page=4&
174Allow: /courses?page=5$
175Allow: /courses?page=5&
176Allow: /courses?*&page=1$
177Allow: /courses?*&page=1&
178Allow: /courses?*&page=2$
179Allow: /courses?*&page=2&
180Allow: /courses?*&page=3$
181Allow: /courses?*&page=3&
182Allow: /courses?*&page=4$
183Allow: /courses?*&page=4&
184Allow: /courses?*&page=5$
185Allow: /courses?*&page=5&
186
187
188User-agent: PerplexityBot
189Disallow: /lecture/
190Disallow: /courses?page=
191Disallow: /courses?*&page=
192Allow: /courses?page=1$
193Allow: /courses?page=1&
194Allow: /courses?page=2$
195Allow: /courses?page=2&
196Allow: /courses?page=3$
197Allow: /courses?page=3&
198Allow: /courses?page=4$
199Allow: /courses?page=4&
200Allow: /courses?page=5$
201Allow: /courses?page=5&
202Allow: /courses?*&page=1$
203Allow: /courses?*&page=1&
204Allow: /courses?*&page=2$
205Allow: /courses?*&page=2&
206Allow: /courses?*&page=3$
207Allow: /courses?*&page=3&
208Allow: /courses?*&page=4$
209Allow: /courses?*&page=4&
210Allow: /courses?*&page=5$
211Allow: /courses?*&page=5&
212
213
214User-agent: LinkedInBot
215Allow: /account/accomplishments/
216Disallow: /courses?page=
217Disallow: /courses?*&page=
218Allow: /courses?page=1$
219Allow: /courses?page=1&
220Allow: /courses?page=2$
221Allow: /courses?page=2&
222Allow: /courses?page=3$
223Allow: /courses?page=3&
224Allow: /courses?page=4$
225Allow: /courses?page=4&
226Allow: /courses?page=5$
227Allow: /courses?page=5&
228Allow: /courses?*&page=1$
229Allow: /courses?*&page=1&
230Allow: /courses?*&page=2$
231Allow: /courses?*&page=2&
232Allow: /courses?*&page=3$
233Allow: /courses?*&page=3&
234Allow: /courses?*&page=4$
235Allow: /courses?*&page=4&
236Allow: /courses?*&page=5$
237Allow: /courses?*&page=5&
238
239
240User-agent: Twitterbot
241Allow: /account/accomplishments/
242Disallow: /courses?page=
243Disallow: /courses?*&page=
244Allow: /courses?page=1$
245Allow: /courses?page=1&
246Allow: /courses?page=2$
247Allow: /courses?page=2&
248Allow: /courses?page=3$
249Allow: /courses?page=3&
250Allow: /courses?page=4$
251Allow: /courses?page=4&
252Allow: /courses?page=5$
253Allow: /courses?page=5&
254Allow: /courses?*&page=1$
255Allow: /courses?*&page=1&
256Allow: /courses?*&page=2$
257Allow: /courses?*&page=2&
258Allow: /courses?*&page=3$
259Allow: /courses?*&page=3&
260Allow: /courses?*&page=4$
261Allow: /courses?*&page=4&
262Allow: /courses?*&page=5$
263Allow: /courses?*&page=5&
264
265
266User-agent: meta-externalagent
267Disallow: /
268
269
270User-agent: OAI-SearchBot
271Allow: /api/utilities/v1/imageproxy
272Disallow: /api/
273Disallow: /courses?page=
274Disallow: /courses?*&page=
275Allow: /courses?page=1$
276Allow: /courses?page=1&
277Allow: /courses?page=2$
278Allow: /courses?page=2&
279Allow: /courses?page=3$
280Allow: /courses?page=3&
281Allow: /courses?page=4$
282Allow: /courses?page=4&
283Allow: /courses?page=5$
284Allow: /courses?page=5&
285Allow: /courses?*&page=1$
286Allow: /courses?*&page=1&
287Allow: /courses?*&page=2$
288Allow: /courses?*&page=2&
289Allow: /courses?*&page=3$
290Allow: /courses?*&page=3&
291Allow: /courses?*&page=4$
292Allow: /courses?*&page=4&
293Allow: /courses?*&page=5$
294Allow: /courses?*&page=5&
295
296
297User-agent: ChatGPT-User
298Allow: /api/utilities/v1/imageproxy
299Disallow: /api/
300Disallow: /courses?page=
301Disallow: /courses?*&page=
302Allow: /courses?page=1$
303Allow: /courses?page=1&
304Allow: /courses?page=2$
305Allow: /courses?page=2&
306Allow: /courses?page=3$
307Allow: /courses?page=3&
308Allow: /courses?page=4$
309Allow: /courses?page=4&
310Allow: /courses?page=5$
311Allow: /courses?page=5&
312Allow: /courses?*&page=1$
313Allow: /courses?*&page=1&
314Allow: /courses?*&page=2$
315Allow: /courses?*&page=2&
316Allow: /courses?*&page=3$
317Allow: /courses?*&page=3&
318Allow: /courses?*&page=4$
319Allow: /courses?*&page=4&
320Allow: /courses?*&page=5$
321Allow: /courses?*&page=5&