OpenAlex / robots.txt
← back to openalex.org · change detected 22d ago (2026-10-16T00:44:51Z) · modified +2 −61
What changed
- 🤖 AIRemoved rules for AI crawler Applebot-Extended (Apple)
- 🤖 AIRemoved rules for AI crawler CCBot (Common Crawl)
- 🤖 AIRemoved rules for AI crawler ClaudeBot (Anthropic)
- 🤖 AIRemoved rules for AI crawler GPTBot (OpenAI)
- 🤖 AIRemoved rules for AI crawler meta-externalagent (Meta)
Diff old (2026-10-15T18:35:45Z) → new (2026-10-16T00:44:51Z)
| @@ -6,7 +6,8 @@ | ||
| 6 | 6 | # out of search indexes with a noindex meta tag, not here. |
| 7 | 7 | # Tests: src/__tests__/robotsTxt.test.js |
| 8 | 8 | |
| 9 | - # Search engines: record pages and company pages; not faceted search or list URLs. | |
| 9 | + # Search engines and AI training crawlers (training allowed: Jason, 2026-10-15): | |
| 10 | + # record pages and company pages; not faceted search or list URLs. | |
| 10 | 11 | User-agent: * |
| 11 | 12 | Disallow: /works? |
| 12 | 13 | Disallow: /authors? |
| @@ -60,64 +61,4 @@ | ||
| 60 | 61 | Disallow: /orcid-callback |
| 61 | 62 | Allow: / |
| 62 | 63 | |
| 63 | - # AI training: company pages yes; record and list pages no. Short record URLs | |
| 64 | - # (/W123) too; lower-case ones that share a letter with a company page (/a123) | |
| 65 | - # redirect to a record path disallowed here. Google-Extended is left out on | |
| 66 | - # purpose: it also controls grounding in Gemini Apps, which is assistant | |
| 67 | - # traffic we want (oxjob #1689). | |
| 68 | - User-agent: GPTBot | |
| 69 | - User-agent: ClaudeBot | |
| 70 | - User-agent: CCBot | |
| 71 | - User-agent: Applebot-Extended | |
| 72 | - User-agent: meta-externalagent | |
| 73 | - Disallow: /works | |
| 74 | - Disallow: /authors | |
| 75 | - Disallow: /sources | |
| 76 | - Disallow: /institutions | |
| 77 | - Disallow: /topics | |
| 78 | - Disallow: /keywords | |
| 79 | - Disallow: /publishers | |
| 80 | - Disallow: /funders | |
| 81 | - Disallow: /awards | |
| 82 | - Disallow: /concepts | |
| 83 | - Disallow: /subfields | |
| 84 | - Disallow: /fields | |
| 85 | - Disallow: /domains | |
| 86 | - Disallow: /sdgs | |
| 87 | - Disallow: /countries | |
| 88 | - Disallow: /continents | |
| 89 | - Disallow: /languages | |
| 90 | - Disallow: /types | |
| 91 | - Disallow: /source-types | |
| 92 | - Disallow: /institution-types | |
| 93 | - Disallow: /licenses | |
| 94 | - Disallow: /oa-statuses | |
| 95 | - Disallow: /indexes | |
| 96 | - Disallow: /source-lists | |
| 97 | - Disallow: /study-designs | |
| 98 | - Disallow: /locations | |
| 99 | - Disallow: /q? | |
| 100 | - Disallow: /q$ | |
| 101 | - Disallow: /W | |
| 102 | - Disallow: /A | |
| 103 | - Disallow: /S | |
| 104 | - Disallow: /I | |
| 105 | - Disallow: /P | |
| 106 | - Disallow: /F | |
| 107 | - Disallow: /C | |
| 108 | - Disallow: /T | |
| 109 | - Disallow: /G | |
| 110 | - Disallow: /w | |
| 111 | - Disallow: /f | |
| 112 | - Disallow: /g | |
| 113 | - Disallow: /admin | |
| 114 | - Disallow: /settings | |
| 115 | - Disallow: /me/ | |
| 116 | - Disallow: /me$ | |
| 117 | - Disallow: /login/magic-token/ | |
| 118 | - Disallow: /oauth/ | |
| 119 | - Disallow: /verify-email | |
| 120 | - Disallow: /orcid-callback | |
| 121 | - Allow: / | |
| 122 | - | |
| 123 | 64 | Sitemap: https://openalex.org/sitemap.xml |