Europe PMC / robots.txt
← back to europepmc.org · change detected 19d ago (2026-07-16T17:46:03Z) · modified +7 −98
What changed
- Removed all rules for user-agent ‘baiduspider’
- Removed all rules for user-agent ‘bingbot’
- 🤖 AIRemoved rules for AI crawler ChatGPT-User (OpenAI)
- 🤖 AIRemoved rules for AI crawler GPTBot (OpenAI)
- 🤖 AIRemoved rules for AI crawler OAI-SearchBot (OpenAI)
- 🤖 AIRemoved rules for AI crawler PerplexityBot (Perplexity)
- Removed all rules for user-agent ‘Perplexity‑User’
- Crawl-delay for *: 5 → 10
- Added Disallow: /advancedsearch for Googlebot
- Removed Disallow: /*/*jsessionid* for Googlebot
- Removed Disallow: /Funders/rss/* for Googlebot
- Removed Disallow: /exception* for Googlebot
- Removed Disallow: /ftp/oa/* for Googlebot
- Removed Disallow: /Funders/rss/* for MSNbot
- Removed Disallow: /abstract/* for MSNbot
- Removed Disallow: /exception/* for MSNbot
- Removed Disallow: /ftp/oa/* for MSNbot
- Removed Disallow: /search?* for MSNbot
- Removed Disallow: /Funders/rss/* for Slurp
- Removed Disallow: /abstract/* for Slurp
- Removed Disallow: /exception/* for Slurp
- Removed Disallow: /ftp/oa/* for Slurp
- Removed Disallow: /search?* for Slurp
- Removed sitemap http://europepmc.org/sitemap_authors_index.xml
- Removed sitemap http://europepmc.org/sitemap_funders_index.xml
- Removed sitemap http://europepmc.org/sitemap_grants_index.xml
- Removed sitemap http://europepmc.org/sitemap_search_index.xml
Diff old (2026-06-26T14:15:22Z) → new (2026-07-16T17:46:03Z)
| @@ -3,15 +3,16 @@ | ||
| 3 | 3 | |
| 4 | 4 | User-agent: Googlebot |
| 5 | 5 | Disallow: /classic |
| 6 | + #Disallow: /*/articles/*/pdf/ | |
| 7 | + #Disallow: /articles/*/pdf/ | |
| 8 | + #Disallow: /articles/*?pdf=render | |
| 9 | + #Disallow: /articles/*?*blobtype=pdf* | |
| 10 | + Disallow: /advancedsearch | |
| 6 | 11 | Disallow: /feedback/* |
| 7 | 12 | Disallow: /*?wicket:interface* |
| 13 | + #Disallow: /search/* | |
| 8 | 14 | Disallow: /*?europe_pmc_extredirect=* |
| 9 | - Disallow: /ftp/oa/* | |
| 10 | - Disallow: /Funders/rss/* | |
| 11 | - Disallow: /exception* | |
| 12 | - Disallow: /*/*jsessionid* | |
| 13 | 15 | |
| 14 | - # | |
| 15 | 16 | # Yahoo |
| 16 | 17 | User-agent: Slurp |
| 17 | 18 | Disallow: /classic |
| @@ -23,24 +24,9 @@ | ||
| 23 | 24 | Disallow: /feedback/* |
| 24 | 25 | Disallow: /*?wicket:interface* |
| 25 | 26 | Disallow: /search/* |
| 26 | - Disallow: /search?* | |
| 27 | - Disallow: /abstract/* | |
| 28 | 27 | Disallow: /*?europe_pmc_extredirect=* |
| 29 | - Disallow: /ftp/oa/* | |
| 30 | - Disallow: /exception/* | |
| 31 | - Disallow: /Funders/rss/* | |
| 32 | 28 | |
| 33 | 29 | # Bing |
| 34 | - User-agent: bingbot | |
| 35 | - Disallow: /classic | |
| 36 | - Disallow: /feedback/* | |
| 37 | - Disallow: /*?wicket:interface* | |
| 38 | - Disallow: /*?europe_pmc_extredirect=* | |
| 39 | - Disallow: /ftp/oa/* | |
| 40 | - Disallow: /Funders/rss/* | |
| 41 | - Disallow: /exception* | |
| 42 | - Disallow: /*/*jsessionid* | |
| 43 | - | |
| 44 | 30 | User-agent: MSNbot |
| 45 | 31 | Disallow: /classic |
| 46 | 32 | Disallow: /*/articles/*/pdf/ |
| @@ -50,81 +36,8 @@ | ||
| 50 | 36 | Disallow: /advancedsearch |
| 51 | 37 | Disallow: /feedback/* |
| 52 | 38 | Disallow: /*?wicket:interface* |
| 53 | - Disallow: /ftp/oa/* | |
| 54 | 39 | Disallow: /search/* |
| 55 | - Disallow: /search?* | |
| 56 | - Disallow: /abstract/* | |
| 57 | 40 | Disallow: /*?europe_pmc_extredirect=* |
| 58 | - Disallow: /Funders/rss/* | |
| 59 | - Disallow: /exception/* | |
| 60 | - | |
| 61 | - User-agent: baiduspider | |
| 62 | - Disallow: /classic | |
| 63 | - Disallow: /*/articles/*/pdf/ | |
| 64 | - Disallow: /articles/*/pdf/ | |
| 65 | - Disallow: /articles/*?pdf=render | |
| 66 | - Disallow: /articles/*?*blobtype=pdf* | |
| 67 | - Disallow: /advancedsearch | |
| 68 | - Disallow: /feedback/* | |
| 69 | - Disallow: /*?wicket:interface* | |
| 70 | - Disallow: /ftp/oa/* | |
| 71 | - Disallow: /search/* | |
| 72 | - Disallow: /search?* | |
| 73 | - Disallow: /abstract/* | |
| 74 | - Disallow: /*?europe_pmc_extredirect=* | |
| 75 | - Disallow: /Funders/rss/* | |
| 76 | - Disallow: /exception/* | |
| 77 | - | |
| 78 | - User-agent: OAI-SearchBot | |
| 79 | - Disallow: /classic | |
| 80 | - Disallow: /feedback/* | |
| 81 | - Disallow: /*?wicket:interface* | |
| 82 | - Disallow: /*?europe_pmc_extredirect=* | |
| 83 | - Disallow: /ftp/oa/* | |
| 84 | - Disallow: /Funders/rss/* | |
| 85 | - Disallow: /exception* | |
| 86 | - Disallow: /*/*jsessionid* | |
| 87 | - | |
| 88 | - User-agent: ChatGPT-User | |
| 89 | - Disallow: /classic | |
| 90 | - Disallow: /feedback/* | |
| 91 | - Disallow: /*?wicket:interface* | |
| 92 | - Disallow: /*?europe_pmc_extredirect=* | |
| 93 | - Disallow: /ftp/oa/* | |
| 94 | - Disallow: /Funders/rss/* | |
| 95 | - Disallow: /exception* | |
| 96 | - Disallow: /*/*jsessionid* | |
| 97 | - | |
| 98 | - User-agent: GPTBot | |
| 99 | - Disallow: /classic | |
| 100 | - Disallow: /feedback/* | |
| 101 | - Disallow: /*?wicket:interface* | |
| 102 | - Disallow: /*?europe_pmc_extredirect=* | |
| 103 | - Disallow: /ftp/oa/* | |
| 104 | - Disallow: /Funders/rss/* | |
| 105 | - Disallow: /exception* | |
| 106 | - Disallow: /*/*jsessionid* | |
| 107 | - | |
| 108 | - User-agent: PerplexityBot | |
| 109 | - Disallow: /classic | |
| 110 | - Disallow: /feedback/* | |
| 111 | - Disallow: /*?wicket:interface* | |
| 112 | - Disallow: /*?europe_pmc_extredirect=* | |
| 113 | - Disallow: /ftp/oa/* | |
| 114 | - Disallow: /Funders/rss/* | |
| 115 | - Disallow: /exception* | |
| 116 | - Disallow: /*/*jsessionid* | |
| 117 | - | |
| 118 | - User-agent: Perplexity‑User | |
| 119 | - Disallow: /classic | |
| 120 | - Disallow: /feedback/* | |
| 121 | - Disallow: /*?wicket:interface* | |
| 122 | - Disallow: /*?europe_pmc_extredirect=* | |
| 123 | - Disallow: /ftp/oa/* | |
| 124 | - Disallow: /Funders/rss/* | |
| 125 | - Disallow: /exception* | |
| 126 | - Disallow: /*/*jsessionid* | |
| 127 | - | |
| 128 | 41 | |
| 129 | 42 | # For all other bots |
| 130 | 43 | User-agent: * |
| @@ -132,10 +45,6 @@ | ||
| 132 | 45 | |
| 133 | 46 | Sitemap: http://europepmc.org/sitemap_abstract_index.xml |
| 134 | 47 | Sitemap: http://europepmc.org/sitemap_fulltext_index.xml |
| 135 | - Sitemap: http://europepmc.org/sitemap_authors_index.xml | |
| 136 | - Sitemap: http://europepmc.org/sitemap_funders_index.xml | |
| 137 | - Sitemap: http://europepmc.org/sitemap_search_index.xml | |
| 138 | - Sitemap: http://europepmc.org/sitemap_grants_index.xml | |
| 139 | - Crawl-delay: 5 | |
| 48 | + Crawl-delay: 10 | |
| 140 | 49 | |
| 141 | 50 |