# openalex.org robots.txt (oxjob #1689). OpenAlex data is free and CC0:
# use the API or the snapshot, https://developers.openalex.org/
#
# A crawler obeys only the most specific group that names it (RFC 9309), so each
# group repeats the rules it needs. Paths are case-sensitive. Thin records are kept
# out of search indexes with a noindex meta tag, not here.
# Tests: src/__tests__/robotsTxt.test.js

# Search engines and AI training crawlers (training allowed: Jason, 2026-10-15):
# record pages and company pages; not faceted search or list URLs.
User-agent: *
Disallow: /works?
Disallow: /authors?
Disallow: /sources?
Disallow: /institutions?
Disallow: /topics?
Disallow: /keywords?
Disallow: /publishers?
Disallow: /funders?
Disallow: /awards?
Disallow: /concepts?
Disallow: /subfields?
Disallow: /fields?
Disallow: /domains?
Disallow: /sdgs?
Disallow: /countries?
Disallow: /continents?
Disallow: /languages?
Disallow: /types?
Disallow: /source-types?
Disallow: /institution-types?
Disallow: /licenses?
Disallow: /oa-statuses?
Disallow: /indexes?
Disallow: /source-lists?
Disallow: /study-designs?
Disallow: /locations?
Disallow: /q?
Disallow: /admin
Disallow: /settings
Disallow: /me/
Disallow: /me$
Disallow: /login/magic-token/
Disallow: /oauth/
Disallow: /verify-email
Disallow: /orcid-callback
Allow: /

# AI assistants fetching a page for a person: everything a person can open.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
Disallow: /admin
Disallow: /settings
Disallow: /me/
Disallow: /me$
Disallow: /login/magic-token/
Disallow: /oauth/
Disallow: /verify-email
Disallow: /orcid-callback
Allow: /

Sitemap: https://openalex.org/sitemap.xml