Coursera / robots.txt

← back to coursera.org · change detected 20d ago (2026-10-17T19:27:16Z) · modified +94 −190

What changed

Added Allow: /courses?*&page=1$ for *
Added Allow: /courses?*&page=1& for *
Added Allow: /courses?*&page=2$ for *
Added Allow: /courses?*&page=2& for *
Added Allow: /courses?*&page=3$ for *
Added Allow: /courses?*&page=3& for *
Added Allow: /courses?*&page=4$ for *
Added Allow: /courses?*&page=4& for *
Added Allow: /courses?*&page=5$ for *
Added Allow: /courses?*&page=5& for *
Added Allow: /courses?page=1$ for *
Added Allow: /courses?page=1& for *
Added Allow: /courses?page=2$ for *
Added Allow: /courses?page=2& for *
Added Allow: /courses?page=3$ for *
Added Allow: /courses?page=3& for *
Added Allow: /courses?page=4$ for *
Added Allow: /courses?page=4& for *
Added Allow: /courses?page=5$ for *
Added Allow: /courses?page=5& for *
Added Disallow: /business/wp-content/uploads/ for *
Added Disallow: /courses?*&page= for *
Added Disallow: /courses?page= for *
Removed Disallow: /business/wp-content/uploads/& for *
Removed Allow: /courses?*&page=1$ for AdsBot-Google
Removed Allow: /courses?*&page=1& for AdsBot-Google
Removed Allow: /courses?*&page=2$ for AdsBot-Google
Removed Allow: /courses?*&page=2& for AdsBot-Google
Removed Allow: /courses?*&page=3$ for AdsBot-Google
Removed Allow: /courses?*&page=3& for AdsBot-Google
Removed Allow: /courses?*&page=4$ for AdsBot-Google
Removed Allow: /courses?*&page=4& for AdsBot-Google
Removed Allow: /courses?*&page=5$ for AdsBot-Google
Removed Allow: /courses?*&page=5& for AdsBot-Google
Removed Allow: /courses?page=1$ for AdsBot-Google
Removed Allow: /courses?page=1& for AdsBot-Google
Removed Allow: /courses?page=2$ for AdsBot-Google
Removed Allow: /courses?page=2& for AdsBot-Google
Removed Allow: /courses?page=3$ for AdsBot-Google
Removed Allow: /courses?page=3& for AdsBot-Google
Removed Allow: /courses?page=4$ for AdsBot-Google
Removed Allow: /courses?page=4& for AdsBot-Google
Removed Allow: /courses?page=5$ for AdsBot-Google
Removed Allow: /courses?page=5& for AdsBot-Google
Removed Disallow: /courses?*&page= for AdsBot-Google
Removed Disallow: /courses?page= for AdsBot-Google
🤖 AIAdded Allow: /api/utilities/v1/imageproxy for CCBot
🤖 AIAdded Allow: /llms.txt for CCBot
🤖 AIAdded Disallow: /5JKLVK/ for CCBot
🤖 AIAdded Disallow: /acclaimbadge/ for CCBot
... and 240 more change(s)

Diff old (2026-09-21T22:21:49Z) → new (2026-10-17T19:27:16Z)

		@@ -18,48 +18,14 @@
18	18	Disallow: /specializations-noperf/
19	19	Disallow: /professional-certificates-noperf/
20	20	Disallow: /business/xmlrpc.php
21		- Disallow: /business/wp-content/uploads/
	21	+ Disallow: /business/wp-content/uploads/
22	22	Disallow: /business/search
23	23	Disallow: /business/teams/search
24	24	Disallow: /organizations/
25	25	Disallow: /programs/
26	26	Disallow: /5JKLVK/
27	27
28		- User-agent: AdsBot-Google
29		- Allow: /
30		-
31		- # Block all /courses pagination
32		- Disallow: /courses?page=
33		- Disallow: /courses?*&page=
34		-
35		- # Allow pagination pages 1-5 when page is first
36		- Allow: /courses?page=1$
37		- Allow: /courses?page=1&
38		- Allow: /courses?page=2$
39		- Allow: /courses?page=2&
40		- Allow: /courses?page=3$
41		- Allow: /courses?page=3&
42		- Allow: /courses?page=4$
43		- Allow: /courses?page=4&
44		- Allow: /courses?page=5$
45		- Allow: /courses?page=5&
46		-
47		- # Allow pagination pages 1-5 when page appears after another parameter
48		- Allow: /courses?*&page=1$
49		- Allow: /courses?*&page=1&
50		- Allow: /courses?*&page=2$
51		- Allow: /courses?*&page=2&
52		- Allow: /courses?*&page=3$
53		- Allow: /courses?*&page=3&
54		- Allow: /courses?*&page=4$
55		- Allow: /courses?*&page=4&
56		- Allow: /courses?*&page=5$
57		- Allow: /courses?*&page=5&
58		-
59		- Sitemap: https://www.coursera.org/sitemap.xml
60		-
61		- User-agent: CCBot
62		- Disallow: /lecture/
	28	+ # /courses pagination: block all, allow pages 1-5
63	29	Disallow: /courses?page=
64	30	Disallow: /courses?*&page=
65	31	Allow: /courses?page=1$
		@@ -83,8 +49,43 @@
83	49	Allow: /courses?*&page=5$
84	50	Allow: /courses?*&page=5&
85	51
	52	+
	53	+ User-agent: AdsBot-Google
	54	+ Allow: /
	55	+
	56	+
	57	+ User-agent: CCBot
86	58	User-agent: GPTBot
	59	+ User-agent: ClaudeBot
	60	+ User-agent: MistralAI-User
	61	+ User-agent: Google-Extended
	62	+ User-agent: PerplexityBot
	63	+ Allow: /api/utilities/v1/imageproxy
	64	+ Allow: /llms.txt
87	65	Disallow: /lecture/
	66	+ Disallow: /maestro/api/
	67	+ Disallow: /api/
	68	+ Disallow: /maestro/
	69	+ Disallow: /ui/
	70	+ Disallow: /signature/voucher/
	71	+ Disallow: /account/
	72	+ Disallow: /acclaimbadge/
	73	+ Disallow: /voucher/
	74	+ Disallow: /search
	75	+ Disallow: /ent-website/
	76	+ Disallow: /learn-perf/
	77	+ Disallow: /specializations-perf/
	78	+ Disallow: /professional-certificates-perf/
	79	+ Disallow: /learn-noperf/
	80	+ Disallow: /specializations-noperf/
	81	+ Disallow: /professional-certificates-noperf/
	82	+ Disallow: /business/xmlrpc.php
	83	+ Disallow: /business/wp-content/uploads/
	84	+ Disallow: /business/search
	85	+ Disallow: /business/teams/search
	86	+ Disallow: /organizations/
	87	+ Disallow: /programs/
	88	+ Disallow: /5JKLVK/
88	89	Disallow: /courses?page=
89	90	Disallow: /courses?*&page=
90	91	Allow: /courses?page=1$
		@@ -108,8 +109,34 @@
108	109	Allow: /courses?*&page=5$
109	110	Allow: /courses?*&page=5&
110	111
111		- User-agent: ClaudeBot
112		- Disallow: /lecture/
	112	+
	113	+ User-agent: OAI-SearchBot
	114	+ User-agent: ChatGPT-User
	115	+ Allow: /api/utilities/v1/imageproxy
	116	+ Allow: /llms.txt
	117	+ Disallow: /maestro/api/
	118	+ Disallow: /api/
	119	+ Disallow: /maestro/
	120	+ Disallow: /ui/
	121	+ Disallow: /signature/voucher/
	122	+ Disallow: /account/
	123	+ Disallow: /acclaimbadge/
	124	+ Disallow: /voucher/
	125	+ Disallow: /search
	126	+ Disallow: /ent-website/
	127	+ Disallow: /learn-perf/
	128	+ Disallow: /specializations-perf/
	129	+ Disallow: /professional-certificates-perf/
	130	+ Disallow: /learn-noperf/
	131	+ Disallow: /specializations-noperf/
	132	+ Disallow: /professional-certificates-noperf/
	133	+ Disallow: /business/xmlrpc.php
	134	+ Disallow: /business/wp-content/uploads/
	135	+ Disallow: /business/search
	136	+ Disallow: /business/teams/search
	137	+ Disallow: /organizations/
	138	+ Disallow: /programs/
	139	+ Disallow: /5JKLVK/
113	140	Disallow: /courses?page=
114	141	Disallow: /courses?*&page=
115	142	Allow: /courses?page=1$
		@@ -133,8 +160,34 @@
133	160	Allow: /courses?*&page=5$
134	161	Allow: /courses?*&page=5&
135	162
136		- User-agent: MistralAI-User
137		- Disallow: /lecture/
	163	+
	164	+ User-agent: LinkedInBot
	165	+ User-agent: Twitterbot
	166	+ Allow: /account/accomplishments/
	167	+ Allow: /api/utilities/v1/imageproxy
	168	+ Disallow: /maestro/api/
	169	+ Disallow: /api/
	170	+ Disallow: /maestro/
	171	+ Disallow: /ui/
	172	+ Disallow: /signature/voucher/
	173	+ Disallow: /account/
	174	+ Disallow: /acclaimbadge/
	175	+ Disallow: /voucher/
	176	+ Disallow: /search
	177	+ Disallow: /ent-website/
	178	+ Disallow: /learn-perf/
	179	+ Disallow: /specializations-perf/
	180	+ Disallow: /professional-certificates-perf/
	181	+ Disallow: /learn-noperf/
	182	+ Disallow: /specializations-noperf/
	183	+ Disallow: /professional-certificates-noperf/
	184	+ Disallow: /business/xmlrpc.php
	185	+ Disallow: /business/wp-content/uploads/
	186	+ Disallow: /business/search
	187	+ Disallow: /business/teams/search
	188	+ Disallow: /organizations/
	189	+ Disallow: /programs/
	190	+ Disallow: /5JKLVK/
138	191	Disallow: /courses?page=
139	192	Disallow: /courses?*&page=
140	193	Allow: /courses?page=1$
		@@ -158,157 +211,8 @@
158	211	Allow: /courses?*&page=5$
159	212	Allow: /courses?*&page=5&
160	213
161		- User-agent: Google-Extended
162		- Disallow: /lecture/
163		- Disallow: /courses?page=
164		- Disallow: /courses?*&page=
165		- Allow: /courses?page=1$
166		- Allow: /courses?page=1&
167		- Allow: /courses?page=2$
168		- Allow: /courses?page=2&
169		- Allow: /courses?page=3$
170		- Allow: /courses?page=3&
171		- Allow: /courses?page=4$
172		- Allow: /courses?page=4&
173		- Allow: /courses?page=5$
174		- Allow: /courses?page=5&
175		- Allow: /courses?*&page=1$
176		- Allow: /courses?*&page=1&
177		- Allow: /courses?*&page=2$
178		- Allow: /courses?*&page=2&
179		- Allow: /courses?*&page=3$
180		- Allow: /courses?*&page=3&
181		- Allow: /courses?*&page=4$
182		- Allow: /courses?*&page=4&
183		- Allow: /courses?*&page=5$
184		- Allow: /courses?*&page=5&
185		-
186		- User-agent: PerplexityBot
187		- Disallow: /lecture/
188		- Disallow: /courses?page=
189		- Disallow: /courses?*&page=
190		- Allow: /courses?page=1$
191		- Allow: /courses?page=1&
192		- Allow: /courses?page=2$
193		- Allow: /courses?page=2&
194		- Allow: /courses?page=3$
195		- Allow: /courses?page=3&
196		- Allow: /courses?page=4$
197		- Allow: /courses?page=4&
198		- Allow: /courses?page=5$
199		- Allow: /courses?page=5&
200		- Allow: /courses?*&page=1$
201		- Allow: /courses?*&page=1&
202		- Allow: /courses?*&page=2$
203		- Allow: /courses?*&page=2&
204		- Allow: /courses?*&page=3$
205		- Allow: /courses?*&page=3&
206		- Allow: /courses?*&page=4$
207		- Allow: /courses?*&page=4&
208		- Allow: /courses?*&page=5$
209		- Allow: /courses?*&page=5&
210		-
211		- User-agent: LinkedInBot
212		- Allow: /account/accomplishments/
213		- Disallow: /courses?page=
214		- Disallow: /courses?*&page=
215		- Allow: /courses?page=1$
216		- Allow: /courses?page=1&
217		- Allow: /courses?page=2$
218		- Allow: /courses?page=2&
219		- Allow: /courses?page=3$
220		- Allow: /courses?page=3&
221		- Allow: /courses?page=4$
222		- Allow: /courses?page=4&
223		- Allow: /courses?page=5$
224		- Allow: /courses?page=5&
225		- Allow: /courses?*&page=1$
226		- Allow: /courses?*&page=1&
227		- Allow: /courses?*&page=2$
228		- Allow: /courses?*&page=2&
229		- Allow: /courses?*&page=3$
230		- Allow: /courses?*&page=3&
231		- Allow: /courses?*&page=4$
232		- Allow: /courses?*&page=4&
233		- Allow: /courses?*&page=5$
234		- Allow: /courses?*&page=5&
235		-
236		- User-agent: Twitterbot
237		- Allow: /account/accomplishments/
238		- Disallow: /courses?page=
239		- Disallow: /courses?*&page=
240		- Allow: /courses?page=1$
241		- Allow: /courses?page=1&
242		- Allow: /courses?page=2$
243		- Allow: /courses?page=2&
244		- Allow: /courses?page=3$
245		- Allow: /courses?page=3&
246		- Allow: /courses?page=4$
247		- Allow: /courses?page=4&
248		- Allow: /courses?page=5$
249		- Allow: /courses?page=5&
250		- Allow: /courses?*&page=1$
251		- Allow: /courses?*&page=1&
252		- Allow: /courses?*&page=2$
253		- Allow: /courses?*&page=2&
254		- Allow: /courses?*&page=3$
255		- Allow: /courses?*&page=3&
256		- Allow: /courses?*&page=4$
257		- Allow: /courses?*&page=4&
258		- Allow: /courses?*&page=5$
259		- Allow: /courses?*&page=5&
260	214
261	215	User-agent: meta-externalagent
262	216	Disallow: /
263	217
264		- User-agent: OAI-SearchBot
265		- Allow: /api/utilities/v1/imageproxy
266		- Disallow: /api/
267		- Disallow: /courses?page=
268		- Disallow: /courses?*&page=
269		- Allow: /courses?page=1$
270		- Allow: /courses?page=1&
271		- Allow: /courses?page=2$
272		- Allow: /courses?page=2&
273		- Allow: /courses?page=3$
274		- Allow: /courses?page=3&
275		- Allow: /courses?page=4$
276		- Allow: /courses?page=4&
277		- Allow: /courses?page=5$
278		- Allow: /courses?page=5&
279		- Allow: /courses?*&page=1$
280		- Allow: /courses?*&page=1&
281		- Allow: /courses?*&page=2$
282		- Allow: /courses?*&page=2&
283		- Allow: /courses?*&page=3$
284		- Allow: /courses?*&page=3&
285		- Allow: /courses?*&page=4$
286		- Allow: /courses?*&page=4&
287		- Allow: /courses?*&page=5$
288		- Allow: /courses?*&page=5&
289		-
290		- User-agent: ChatGPT-User
291		- Allow: /api/utilities/v1/imageproxy
292		- Disallow: /api/
293		- Disallow: /courses?page=
294		- Disallow: /courses?*&page=
295		- Allow: /courses?page=1$
296		- Allow: /courses?page=1&
297		- Allow: /courses?page=2$
298		- Allow: /courses?page=2&
299		- Allow: /courses?page=3$
300		- Allow: /courses?page=3&
301		- Allow: /courses?page=4$
302		- Allow: /courses?page=4&
303		- Allow: /courses?page=5$
304		- Allow: /courses?page=5&
305		- Allow: /courses?*&page=1$
306		- Allow: /courses?*&page=1&
307		- Allow: /courses?*&page=2$
308		- Allow: /courses?*&page=2&
309		- Allow: /courses?*&page=3$
310		- Allow: /courses?*&page=3&
311		- Allow: /courses?*&page=4$
312		- Allow: /courses?*&page=4&
313		- Allow: /courses?*&page=5$
314		- Allow: /courses?*&page=5&
	218	+ Sitemap: https://www.coursera.org/sitemap.xml