Coursera / robots.txt
← back to coursera.org · change detected 1mo ago (2026-09-21T22:21:49Z) · modified +4 −11
What changed
- Added rules for user-agent ‘AdsBot-Google’
- Added Disallow: /5JKLVK/ for *
- Removed Allow: /courses?*&page=1$ for *
- Removed Allow: /courses?*&page=1& for *
- Removed Allow: /courses?*&page=2$ for *
- Removed Allow: /courses?*&page=2& for *
- Removed Allow: /courses?*&page=3$ for *
- Removed Allow: /courses?*&page=3& for *
- Removed Allow: /courses?*&page=4$ for *
- Removed Allow: /courses?*&page=4& for *
- Removed Allow: /courses?*&page=5$ for *
- Removed Allow: /courses?*&page=5& for *
- Removed Allow: /courses?page=1$ for *
- Removed Allow: /courses?page=1& for *
- Removed Allow: /courses?page=2$ for *
- Removed Allow: /courses?page=2& for *
- Removed Allow: /courses?page=3$ for *
- Removed Allow: /courses?page=3& for *
- Removed Allow: /courses?page=4$ for *
- Removed Allow: /courses?page=4& for *
- Removed Allow: /courses?page=5$ for *
- Removed Allow: /courses?page=5& for *
- Removed Disallow: /courses?*&page= for *
- Removed Disallow: /courses?page= for *
Diff old (2026-09-11T00:58:05Z) → new (2026-09-21T22:21:49Z)
| @@ -23,6 +23,10 @@ | ||
| 23 | 23 | Disallow: /business/teams/search |
| 24 | 24 | Disallow: /organizations/ |
| 25 | 25 | Disallow: /programs/ |
| 26 | + Disallow: /5JKLVK/ | |
| 27 | + | |
| 28 | + User-agent: AdsBot-Google | |
| 29 | + Allow: / | |
| 26 | 30 | |
| 27 | 31 | # Block all /courses pagination |
| 28 | 32 | Disallow: /courses?page= |
| @@ -54,7 +58,6 @@ | ||
| 54 | 58 | |
| 55 | 59 | Sitemap: https://www.coursera.org/sitemap.xml |
| 56 | 60 | |
| 57 | - | |
| 58 | 61 | User-agent: CCBot |
| 59 | 62 | Disallow: /lecture/ |
| 60 | 63 | Disallow: /courses?page= |
| @@ -79,7 +82,6 @@ | ||
| 79 | 82 | Allow: /courses?*&page=4& |
| 80 | 83 | Allow: /courses?*&page=5$ |
| 81 | 84 | Allow: /courses?*&page=5& |
| 82 | - | |
| 83 | 85 | |
| 84 | 86 | User-agent: GPTBot |
| 85 | 87 | Disallow: /lecture/ |
| @@ -106,7 +108,6 @@ | ||
| 106 | 108 | Allow: /courses?*&page=5$ |
| 107 | 109 | Allow: /courses?*&page=5& |
| 108 | 110 | |
| 109 | - | |
| 110 | 111 | User-agent: ClaudeBot |
| 111 | 112 | Disallow: /lecture/ |
| 112 | 113 | Disallow: /courses?page= |
| @@ -131,7 +132,6 @@ | ||
| 131 | 132 | Allow: /courses?*&page=4& |
| 132 | 133 | Allow: /courses?*&page=5$ |
| 133 | 134 | Allow: /courses?*&page=5& |
| 134 | - | |
| 135 | 135 | |
| 136 | 136 | User-agent: MistralAI-User |
| 137 | 137 | Disallow: /lecture/ |
| @@ -158,7 +158,6 @@ | ||
| 158 | 158 | Allow: /courses?*&page=5$ |
| 159 | 159 | Allow: /courses?*&page=5& |
| 160 | 160 | |
| 161 | - | |
| 162 | 161 | User-agent: Google-Extended |
| 163 | 162 | Disallow: /lecture/ |
| 164 | 163 | Disallow: /courses?page= |
| @@ -183,7 +182,6 @@ | ||
| 183 | 182 | Allow: /courses?*&page=4& |
| 184 | 183 | Allow: /courses?*&page=5$ |
| 185 | 184 | Allow: /courses?*&page=5& |
| 186 | - | |
| 187 | 185 | |
| 188 | 186 | User-agent: PerplexityBot |
| 189 | 187 | Disallow: /lecture/ |
| @@ -210,7 +208,6 @@ | ||
| 210 | 208 | Allow: /courses?*&page=5$ |
| 211 | 209 | Allow: /courses?*&page=5& |
| 212 | 210 | |
| 213 | - | |
| 214 | 211 | User-agent: LinkedInBot |
| 215 | 212 | Allow: /account/accomplishments/ |
| 216 | 213 | Disallow: /courses?page= |
| @@ -235,7 +232,6 @@ | ||
| 235 | 232 | Allow: /courses?*&page=4& |
| 236 | 233 | Allow: /courses?*&page=5$ |
| 237 | 234 | Allow: /courses?*&page=5& |
| 238 | - | |
| 239 | 235 | |
| 240 | 236 | User-agent: Twitterbot |
| 241 | 237 | Allow: /account/accomplishments/ |
| @@ -262,10 +258,8 @@ | ||
| 262 | 258 | Allow: /courses?*&page=5$ |
| 263 | 259 | Allow: /courses?*&page=5& |
| 264 | 260 | |
| 265 | - | |
| 266 | 261 | User-agent: meta-externalagent |
| 267 | 262 | Disallow: / |
| 268 | - | |
| 269 | 263 | |
| 270 | 264 | User-agent: OAI-SearchBot |
| 271 | 265 | Allow: /api/utilities/v1/imageproxy |
| @@ -293,7 +287,6 @@ | ||
| 293 | 287 | Allow: /courses?*&page=5$ |
| 294 | 288 | Allow: /courses?*&page=5& |
| 295 | 289 | |
| 296 | - | |
| 297 | 290 | User-agent: ChatGPT-User |
| 298 | 291 | Allow: /api/utilities/v1/imageproxy |
| 299 | 292 | Disallow: /api/ |