NeuralCrawl

Europe PMC / robots.txt snapshot

← back to europepmc.org · fetched 2026-08-28T20:43:06Z (21d ago) · HTTP 200 · 5118 bytes · sha256 6a34da8a75a841d5 · raw

final URL: https://europepmc.org/robots.txt

1# As a condition of accessing this website, you agree to abide by the following
2# content signals:
3
4# (a) If a Content-Signal = yes, you may collect content for the corresponding
5# use.
6# (b) If a Content-Signal = no, you may not collect content for the
7# corresponding use.
8# (c) If the website operator does not include a Content-Signal for a
9# corresponding use, the website operator neither grants nor restricts
10# permission via Content-Signal with respect to the corresponding use.
11
12# The content signals and their meanings are:
13
14# search: building a search index and providing search results (e.g., returning
15# hyperlinks and short excerpts from your website's contents). Search does not
16# include providing AI-generated search summaries.
17# ai-input: inputting content into one or more AI models (e.g., retrieval
18# augmented generation, grounding, or other real-time taking of content for
19# generative AI search answers).
20# ai-train: training or fine-tuning AI models.
21# use: how AI systems may consume the content (immediate, reference, or full).
22
23# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
24# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
25# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
26
27# BEGIN Cloudflare Managed content
28
29User-agent: *
30Content-Signal: search=yes,ai-train=no,use=reference
31Allow: /
32
33User-agent: Amazonbot
34Disallow: /
35
36User-agent: Applebot-Extended
37Disallow: /
38
39User-agent: Bytespider
40Disallow: /
41
42User-agent: CCBot
43Disallow: /
44
45User-agent: ClaudeBot
46Disallow: /
47
48User-agent: CloudflareBrowserRenderingCrawler
49Disallow: /
50
51User-agent: Google-Extended
52Disallow: /
53
54User-agent: GPTBot
55Disallow: /
56
57User-agent: meta-externalagent
58Disallow: /
59
60# END Cloudflare Managed Content
61
62# UKPMC robots.txt
63
64# Google
65User-agent: Googlebot
66Disallow: /classic
67Disallow: /feedback/*
68Disallow: /*?wicket:interface*
69Disallow: /*?europe_pmc_extredirect=*
70Disallow: /ftp/oa/*
71Disallow: /Funders/rss/*
72Disallow: /exception*
73Disallow: /*/*jsessionid*
74
75#
76# Yahoo
77User-agent: Slurp
78Disallow: /classic
79Disallow: /*/articles/*/pdf/
80Disallow: /articles/*/pdf/
81Disallow: /articles/*?pdf=render
82Disallow: /articles/*?*blobtype=pdf*
83Disallow: /advancedsearch
84Disallow: /feedback/*
85Disallow: /*?wicket:interface*
86Disallow: /search/*
87Disallow: /search?*
88Disallow: /abstract/*
89Disallow: /*?europe_pmc_extredirect=*
90Disallow: /ftp/oa/*
91Disallow: /exception/*
92Disallow: /Funders/rss/*
93
94# Bing
95User-agent: bingbot
96Disallow: /classic
97Disallow: /feedback/*
98Disallow: /*?wicket:interface*
99Disallow: /*?europe_pmc_extredirect=*
100Disallow: /ftp/oa/*
101Disallow: /Funders/rss/*
102Disallow: /exception*
103Disallow: /*/*jsessionid*
104
105User-agent: MSNbot
106Disallow: /classic
107Disallow: /*/articles/*/pdf/
108Disallow: /articles/*/pdf/
109Disallow: /articles/*?pdf=render
110Disallow: /articles/*?*blobtype=pdf*
111Disallow: /advancedsearch
112Disallow: /feedback/*
113Disallow: /*?wicket:interface*
114Disallow: /ftp/oa/*
115Disallow: /search/*
116Disallow: /search?*
117Disallow: /abstract/*
118Disallow: /*?europe_pmc_extredirect=*
119Disallow: /Funders/rss/*
120Disallow: /exception/*
121
122User-agent: baiduspider
123Disallow: /classic
124Disallow: /*/articles/*/pdf/
125Disallow: /articles/*/pdf/
126Disallow: /articles/*?pdf=render
127Disallow: /articles/*?*blobtype=pdf*
128Disallow: /advancedsearch
129Disallow: /feedback/*
130Disallow: /*?wicket:interface*
131Disallow: /ftp/oa/*
132Disallow: /search/*
133Disallow: /search?*
134Disallow: /abstract/*
135Disallow: /*?europe_pmc_extredirect=*
136Disallow: /Funders/rss/*
137Disallow: /exception/*
138
139User-agent: OAI-SearchBot
140Disallow: /classic
141Disallow: /feedback/*
142Disallow: /*?wicket:interface*
143Disallow: /*?europe_pmc_extredirect=*
144Disallow: /ftp/oa/*
145Disallow: /Funders/rss/*
146Disallow: /exception*
147Disallow: /*/*jsessionid*
148
149User-agent: ChatGPT-User
150Disallow: /classic
151Disallow: /feedback/*
152Disallow: /*?wicket:interface*
153Disallow: /*?europe_pmc_extredirect=*
154Disallow: /ftp/oa/*
155Disallow: /Funders/rss/*
156Disallow: /exception*
157Disallow: /*/*jsessionid*
158
159User-agent: GPTBot
160Disallow: /classic
161Disallow: /feedback/*
162Disallow: /*?wicket:interface*
163Disallow: /*?europe_pmc_extredirect=*
164Disallow: /ftp/oa/*
165Disallow: /Funders/rss/*
166Disallow: /exception*
167Disallow: /*/*jsessionid*
168
169User-agent: PerplexityBot
170Disallow: /classic
171Disallow: /feedback/*
172Disallow: /*?wicket:interface*
173Disallow: /*?europe_pmc_extredirect=*
174Disallow: /ftp/oa/*
175Disallow: /Funders/rss/*
176Disallow: /exception*
177Disallow: /*/*jsessionid*
178
179User-agent: Perplexity‑User
180Disallow: /classic
181Disallow: /feedback/*
182Disallow: /*?wicket:interface*
183Disallow: /*?europe_pmc_extredirect=*
184Disallow: /ftp/oa/*
185Disallow: /Funders/rss/*
186Disallow: /exception*
187Disallow: /*/*jsessionid*
188
189
190# For all other bots
191User-agent: *
192Disallow: /
193
194Sitemap: http://europepmc.org/sitemap_abstract_index.xml
195Sitemap: http://europepmc.org/sitemap_fulltext_index.xml
196Sitemap: http://europepmc.org/sitemap_authors_index.xml
197Sitemap: http://europepmc.org/sitemap_funders_index.xml
198Sitemap: http://europepmc.org/sitemap_search_index.xml
199Sitemap: http://europepmc.org/sitemap_grants_index.xml
200Crawl-delay: 5
201
202