NeuralCrawl

Coursera / robots.txt snapshot

← back to coursera.org · fetched 2026-09-21T22:21:49Z (1mo ago) · HTTP 200 · 7577 bytes · sha256 d122e2845060386f · raw

final URL: https://www.coursera.org/robots.txt

1User-agent: *
2Allow: /api/utilities/v1/imageproxy
3Allow: /llms.txt
4Disallow: /maestro/api/
5Disallow: /api/
6Disallow: /maestro/
7Disallow: /ui/
8Disallow: /signature/voucher/
9Disallow: /account/
10Disallow: /acclaimbadge/
11Disallow: /voucher/
12Disallow: /search
13Disallow: /ent-website/
14Disallow: /learn-perf/
15Disallow: /specializations-perf/
16Disallow: /professional-certificates-perf/
17Disallow: /learn-noperf/
18Disallow: /specializations-noperf/
19Disallow: /professional-certificates-noperf/
20Disallow: /business/xmlrpc.php
21Disallow: /business/wp-content/uploads/ 
22Disallow: /business/search
23Disallow: /business/teams/search
24Disallow: /organizations/
25Disallow: /programs/
26Disallow: /5JKLVK/
27
28User-agent: AdsBot-Google
29Allow: /
30
31# Block all /courses pagination
32Disallow: /courses?page=
33Disallow: /courses?*&page=
34
35# Allow pagination pages 1-5 when page is first
36Allow: /courses?page=1$
37Allow: /courses?page=1&
38Allow: /courses?page=2$
39Allow: /courses?page=2&
40Allow: /courses?page=3$
41Allow: /courses?page=3&
42Allow: /courses?page=4$
43Allow: /courses?page=4&
44Allow: /courses?page=5$
45Allow: /courses?page=5&
46
47# Allow pagination pages 1-5 when page appears after another parameter
48Allow: /courses?*&page=1$
49Allow: /courses?*&page=1&
50Allow: /courses?*&page=2$
51Allow: /courses?*&page=2&
52Allow: /courses?*&page=3$
53Allow: /courses?*&page=3&
54Allow: /courses?*&page=4$
55Allow: /courses?*&page=4&
56Allow: /courses?*&page=5$
57Allow: /courses?*&page=5&
58
59Sitemap: https://www.coursera.org/sitemap.xml
60
61User-agent: CCBot
62Disallow: /lecture/
63Disallow: /courses?page=
64Disallow: /courses?*&page=
65Allow: /courses?page=1$
66Allow: /courses?page=1&
67Allow: /courses?page=2$
68Allow: /courses?page=2&
69Allow: /courses?page=3$
70Allow: /courses?page=3&
71Allow: /courses?page=4$
72Allow: /courses?page=4&
73Allow: /courses?page=5$
74Allow: /courses?page=5&
75Allow: /courses?*&page=1$
76Allow: /courses?*&page=1&
77Allow: /courses?*&page=2$
78Allow: /courses?*&page=2&
79Allow: /courses?*&page=3$
80Allow: /courses?*&page=3&
81Allow: /courses?*&page=4$
82Allow: /courses?*&page=4&
83Allow: /courses?*&page=5$
84Allow: /courses?*&page=5&
85
86User-agent: GPTBot
87Disallow: /lecture/
88Disallow: /courses?page=
89Disallow: /courses?*&page=
90Allow: /courses?page=1$
91Allow: /courses?page=1&
92Allow: /courses?page=2$
93Allow: /courses?page=2&
94Allow: /courses?page=3$
95Allow: /courses?page=3&
96Allow: /courses?page=4$
97Allow: /courses?page=4&
98Allow: /courses?page=5$
99Allow: /courses?page=5&
100Allow: /courses?*&page=1$
101Allow: /courses?*&page=1&
102Allow: /courses?*&page=2$
103Allow: /courses?*&page=2&
104Allow: /courses?*&page=3$
105Allow: /courses?*&page=3&
106Allow: /courses?*&page=4$
107Allow: /courses?*&page=4&
108Allow: /courses?*&page=5$
109Allow: /courses?*&page=5&
110
111User-agent: ClaudeBot
112Disallow: /lecture/
113Disallow: /courses?page=
114Disallow: /courses?*&page=
115Allow: /courses?page=1$
116Allow: /courses?page=1&
117Allow: /courses?page=2$
118Allow: /courses?page=2&
119Allow: /courses?page=3$
120Allow: /courses?page=3&
121Allow: /courses?page=4$
122Allow: /courses?page=4&
123Allow: /courses?page=5$
124Allow: /courses?page=5&
125Allow: /courses?*&page=1$
126Allow: /courses?*&page=1&
127Allow: /courses?*&page=2$
128Allow: /courses?*&page=2&
129Allow: /courses?*&page=3$
130Allow: /courses?*&page=3&
131Allow: /courses?*&page=4$
132Allow: /courses?*&page=4&
133Allow: /courses?*&page=5$
134Allow: /courses?*&page=5&
135
136User-agent: MistralAI-User
137Disallow: /lecture/
138Disallow: /courses?page=
139Disallow: /courses?*&page=
140Allow: /courses?page=1$
141Allow: /courses?page=1&
142Allow: /courses?page=2$
143Allow: /courses?page=2&
144Allow: /courses?page=3$
145Allow: /courses?page=3&
146Allow: /courses?page=4$
147Allow: /courses?page=4&
148Allow: /courses?page=5$
149Allow: /courses?page=5&
150Allow: /courses?*&page=1$
151Allow: /courses?*&page=1&
152Allow: /courses?*&page=2$
153Allow: /courses?*&page=2&
154Allow: /courses?*&page=3$
155Allow: /courses?*&page=3&
156Allow: /courses?*&page=4$
157Allow: /courses?*&page=4&
158Allow: /courses?*&page=5$
159Allow: /courses?*&page=5&
160
161User-agent: Google-Extended
162Disallow: /lecture/
163Disallow: /courses?page=
164Disallow: /courses?*&page=
165Allow: /courses?page=1$
166Allow: /courses?page=1&
167Allow: /courses?page=2$
168Allow: /courses?page=2&
169Allow: /courses?page=3$
170Allow: /courses?page=3&
171Allow: /courses?page=4$
172Allow: /courses?page=4&
173Allow: /courses?page=5$
174Allow: /courses?page=5&
175Allow: /courses?*&page=1$
176Allow: /courses?*&page=1&
177Allow: /courses?*&page=2$
178Allow: /courses?*&page=2&
179Allow: /courses?*&page=3$
180Allow: /courses?*&page=3&
181Allow: /courses?*&page=4$
182Allow: /courses?*&page=4&
183Allow: /courses?*&page=5$
184Allow: /courses?*&page=5&
185
186User-agent: PerplexityBot
187Disallow: /lecture/
188Disallow: /courses?page=
189Disallow: /courses?*&page=
190Allow: /courses?page=1$
191Allow: /courses?page=1&
192Allow: /courses?page=2$
193Allow: /courses?page=2&
194Allow: /courses?page=3$
195Allow: /courses?page=3&
196Allow: /courses?page=4$
197Allow: /courses?page=4&
198Allow: /courses?page=5$
199Allow: /courses?page=5&
200Allow: /courses?*&page=1$
201Allow: /courses?*&page=1&
202Allow: /courses?*&page=2$
203Allow: /courses?*&page=2&
204Allow: /courses?*&page=3$
205Allow: /courses?*&page=3&
206Allow: /courses?*&page=4$
207Allow: /courses?*&page=4&
208Allow: /courses?*&page=5$
209Allow: /courses?*&page=5&
210
211User-agent: LinkedInBot
212Allow: /account/accomplishments/
213Disallow: /courses?page=
214Disallow: /courses?*&page=
215Allow: /courses?page=1$
216Allow: /courses?page=1&
217Allow: /courses?page=2$
218Allow: /courses?page=2&
219Allow: /courses?page=3$
220Allow: /courses?page=3&
221Allow: /courses?page=4$
222Allow: /courses?page=4&
223Allow: /courses?page=5$
224Allow: /courses?page=5&
225Allow: /courses?*&page=1$
226Allow: /courses?*&page=1&
227Allow: /courses?*&page=2$
228Allow: /courses?*&page=2&
229Allow: /courses?*&page=3$
230Allow: /courses?*&page=3&
231Allow: /courses?*&page=4$
232Allow: /courses?*&page=4&
233Allow: /courses?*&page=5$
234Allow: /courses?*&page=5&
235
236User-agent: Twitterbot
237Allow: /account/accomplishments/
238Disallow: /courses?page=
239Disallow: /courses?*&page=
240Allow: /courses?page=1$
241Allow: /courses?page=1&
242Allow: /courses?page=2$
243Allow: /courses?page=2&
244Allow: /courses?page=3$
245Allow: /courses?page=3&
246Allow: /courses?page=4$
247Allow: /courses?page=4&
248Allow: /courses?page=5$
249Allow: /courses?page=5&
250Allow: /courses?*&page=1$
251Allow: /courses?*&page=1&
252Allow: /courses?*&page=2$
253Allow: /courses?*&page=2&
254Allow: /courses?*&page=3$
255Allow: /courses?*&page=3&
256Allow: /courses?*&page=4$
257Allow: /courses?*&page=4&
258Allow: /courses?*&page=5$
259Allow: /courses?*&page=5&
260
261User-agent: meta-externalagent
262Disallow: /
263
264User-agent: OAI-SearchBot
265Allow: /api/utilities/v1/imageproxy
266Disallow: /api/
267Disallow: /courses?page=
268Disallow: /courses?*&page=
269Allow: /courses?page=1$
270Allow: /courses?page=1&
271Allow: /courses?page=2$
272Allow: /courses?page=2&
273Allow: /courses?page=3$
274Allow: /courses?page=3&
275Allow: /courses?page=4$
276Allow: /courses?page=4&
277Allow: /courses?page=5$
278Allow: /courses?page=5&
279Allow: /courses?*&page=1$
280Allow: /courses?*&page=1&
281Allow: /courses?*&page=2$
282Allow: /courses?*&page=2&
283Allow: /courses?*&page=3$
284Allow: /courses?*&page=3&
285Allow: /courses?*&page=4$
286Allow: /courses?*&page=4&
287Allow: /courses?*&page=5$
288Allow: /courses?*&page=5&
289
290User-agent: ChatGPT-User
291Allow: /api/utilities/v1/imageproxy
292Disallow: /api/
293Disallow: /courses?page=
294Disallow: /courses?*&page=
295Allow: /courses?page=1$
296Allow: /courses?page=1&
297Allow: /courses?page=2$
298Allow: /courses?page=2&
299Allow: /courses?page=3$
300Allow: /courses?page=3&
301Allow: /courses?page=4$
302Allow: /courses?page=4&
303Allow: /courses?page=5$
304Allow: /courses?page=5&
305Allow: /courses?*&page=1$
306Allow: /courses?*&page=1&
307Allow: /courses?*&page=2$
308Allow: /courses?*&page=2&
309Allow: /courses?*&page=3$
310Allow: /courses?*&page=3&
311Allow: /courses?*&page=4$
312Allow: /courses?*&page=4&
313Allow: /courses?*&page=5$
314Allow: /courses?*&page=5&