NeuralCrawl

PubMed / NCBI / robots.txt snapshot

← back to ncbi.nlm.nih.gov · fetched 2026-06-26T14:15:22Z (4h ago) · HTTP 200 · 10820 bytes · sha256 93370c444f0df814 · raw

final URL: https://ncbi.nlm.nih.gov/robots.txt

1# robots.txt - robot exclusion file - back-end server version - no robots!
2# ========================================================================
3User-agent: htdig/3.1.6
4Disallow: /cgi-bin
5Disallow: /entrez
6Disallow: /stat
7Disallow: /COG
8Disallow: /Entrez
9Disallow: /mailman
10Disallow: /myncbi
11Disallow: /sutils
12Disallow: /Taxonomy/Selector
13Disallow: /Taxonomy/CommonTree
14Disallow: /entrez/sutils
15Disallow: /mapview
16Disallow: /blast/BlastAlign.cgi
17Disallow: /blast/bl2seq/wblast2.cgi
18Disallow: /portal
19Disallow: /pmc/utilities/
20Disallow: /pmc/issues/
21Disallow: /pmc/articles/
22Disallow: /pmc/ivip/
23Disallow: /pmc/articlerender.fcgi
24Disallow: /pmc/pagerender.fcgi
25Disallow: /pmc/picrender.fcgi
26Disallow: /pmc/tocrender.fcgi
27Disallow: /pmc/*report=
28Disallow: /pmc/*page=
29Disallow: /pmc/articles/*/citedby/
30Disallow: /pmc/publisherportal/api/
31Disallow: /pmc/publisherportal/application/
32Disallow: /pmc/publisherportal/download
33Disallow: /pmc/publisherportal/journalmanager/
34Disallow: /labs/pmc/
35Disallow: /geo/tools
36Disallow: /geo/browse
37Disallow: /projects/geo/tools
38Disallow: /geo/download
39Disallow: /projects/geo/download
40Disallow: /projects/gtr/gt
41Disallow: /Structure/flink/flink.cgi
42Disallow: /Structure/vast/vastsrv.cgi
43Disallow: /viewvc
44Disallow: /sars-cov-2
45
46User-agent: Baiduspider
47Disallow: /cgi-bin
48Disallow: /entrez
49Disallow: /stat
50Disallow: /COG
51Disallow: /Entrez
52Disallow: /mailman
53Disallow: /myncbi
54Disallow: /sutils
55Disallow: /Taxonomy/Selector
56Disallow: /Taxonomy/CommonTree
57Disallow: /entrez/sutils
58Disallow: /mapview
59Disallow: /blast/BlastAlign.cgi
60Disallow: /blast/bl2seq/wblast2.cgi
61Disallow: /portal
62Disallow: /pmc/utilities/
63Disallow: /pmc/issues/
64Disallow: /pmc/articles/
65Disallow: /pmc/ivip/
66Disallow: /pmc/articlerender.fcgi
67Disallow: /pmc/pagerender.fcgi
68Disallow: /pmc/picrender.fcgi
69Disallow: /pmc/tocrender.fcgi
70Disallow: /pmc/*report=
71Disallow: /pmc/*page=
72Disallow: /pmc/articles/*/citedby/
73Disallow: /pmc/publisherportal/api/
74Disallow: /pmc/publisherportal/application/
75Disallow: /pmc/publisherportal/download
76Disallow: /pmc/publisherportal/journalmanager/
77Disallow: /labs/pmc/
78Disallow: /geo/tools
79Disallow: /geo/browse
80Disallow: /projects/geo/tools
81Disallow: /geo/download
82Disallow: /projects/geo/download
83Disallow: /projects/gtr/gt
84Disallow: /Structure/flink/flink.cgi
85Disallow: /Structure/vast/vastsrv.cgi
86Disallow: /viewvc
87Disallow: /sars-cov-2
88
89User-agent: Googlebot
90Disallow: /cgi-bin
91Disallow: /stat
92Disallow: /COG
93Disallow: /mailman
94Disallow: /sutils
95Disallow: /entrez/viewer.fcgi
96Disallow: /Taxonomy/Selector
97Disallow: /Taxonomy/CommonTree
98Disallow: /entrez/sutils
99Disallow: /SAGE
100Disallow: /SNP
101Disallow: /snp
102Disallow: /IEB/ToolBox/SB
103Disallow: /Structure/dgs
104Disallow: /Structure/VA
105Disallow: /Structure/cblast/cblast.cgi
106Disallow: /Coffeebreak
107Disallow: /gorf
108Disallow: /mapview
109Disallow: /myncbi
110Disallow: /blast/BlastAlign.cgi
111Disallow: /blast/bl2seq/wblast2.cgi
112Disallow: /entrez/eutils
113Disallow: /entrez/utils
114Disallow: /portal
115Disallow: /pmc/utilities/
116Disallow: /pmc/issues/
117Disallow: /pmc/*/bin/
118Disallow: /pmc/*/pdf/
119Disallow: /pmc/*/equ/
120Disallow: /pmc/*.cgi
121Disallow: /pmc/*report=
122Disallow: /pmc/*page=
123Disallow: /pmc/articles/*/citedby/
124Disallow: /pmc/publisherportal/api/
125Disallow: /pmc/publisherportal/application/
126Disallow: /pmc/publisherportal/download
127Disallow: /pmc/publisherportal/journalmanager/
128Disallow: /labs/pmc/
129Disallow: /geo/tools
130Disallow: /geo/browse
131Disallow: /projects/geo/tools
132Disallow: /geo/download
133Disallow: /projects/geo/download
134Disallow: /projects/gtr/gt
135Disallow: /Structure/flink/flink.cgi
136Disallow: /Structure/vast/vastsrv.cgi
137Disallow: /viewvc
138Disallow: /clone/library/genomic/organisms
139Disallow: /sars-cov-2
140Allow: /portal/portal3rc.fcgi/*/*.js
141Allow: /portal/portal3rc.fcgi/*/*.css
142
143User-agent: Teleport Pro
144Disallow: /cgi-bin
145Disallow: /entrez
146Disallow: /stat
147Disallow: /COG
148Disallow: /Entrez
149Disallow: /myncbi
150Disallow: /mailman
151Disallow: /sutils
152Disallow: /Taxonomy/Selector
153Disallow: /Taxonomy/CommonTree
154Disallow: /entrez/sutils
155Disallow: /mapview
156Disallow: /blast/BlastAlign.cgi
157Disallow: /blast/bl2seq/wblast2.cgi
158Disallow: /portal
159Disallow: /pmc/utilities/
160Disallow: /pmc/issues/
161Disallow: /pmc/articles/
162Disallow: /pmc/ivip/
163Disallow: /pmc/articlerender.fcgi
164Disallow: /pmc/pagerender.fcgi
165Disallow: /pmc/picrender.fcgi
166Disallow: /pmc/tocrender.fcgi
167Disallow: /pmc/publisherportal/api/
168Disallow: /pmc/publisherportal/application/
169Disallow: /pmc/publisherportal/download
170Disallow: /pmc/publisherportal/journalmanager/
171Disallow: /labs/pmc/
172Disallow: /geo/tools
173Disallow: /geo/browse
174Disallow: /projects/geo/tools
175Disallow: /geo/download
176Disallow: /projects/geo/download
177Disallow: /projects/gtr/gt
178Disallow: /Structure/flink/flink.cgi
179Disallow: /Structure/vast/vastsrv.cgi
180Disallow: /viewvc
181Disallow: /sars-cov-2
182
183User-agent: yacybot
184Disallow: *
185
186User-agent: Browsershots
187Disallow:
188Disallow: /sars-cov-2
189
190User-agent: Slurp
191Crawl-delay: 1
192Disallow: /cgi-bin
193Disallow: /entrez
194Disallow: /stat
195Disallow: /COG
196Disallow: /Entrez
197Disallow: /myncbi
198Disallow: /mailman
199Disallow: /sutils
200Disallow: /Taxonomy/Selector
201Disallow: /Taxonomy/CommonTree
202Disallow: /entrez/sutils
203Disallow: /mapview
204Disallow: /blast/BlastAlign.cgi
205Disallow: /blast/bl2seq/wblast2.cgi
206Disallow: /SAGE
207Disallow: /SNP
208Disallow: /snp
209Disallow: /portal
210Disallow: /pmc/utilities/
211Disallow: /pmc/*/bin/
212Disallow: /pmc/*/pdf/
213Disallow: /pmc/*/equ/
214Disallow: /pmc/*.cgi
215Disallow: /pmc/*report=
216Disallow: /pmc/*page=
217Disallow: /pmc/articles/*/citedby/
218Disallow: /pmc/publisherportal/api/
219Disallow: /pmc/publisherportal/application/
220Disallow: /pmc/publisherportal/download
221Disallow: /pmc/publisherportal/journalmanager/
222Disallow: /labs/pmc/
223Disallow: /geo/tools
224Disallow: /geo/browse
225Disallow: /projects/geo/tools
226Disallow: /geo/download
227Disallow: /projects/geo/download
228Disallow: /projects/gtr/gt
229Disallow: /Structure/flink/flink.cgi
230Disallow: /Structure/vast/vastsrv.cgi
231Disallow: /viewvc
232Disallow: /sars-cov-2
233
234User-agent: Twitterbot
235Disallow: *
236Allow: /pmc/articles
237Allow: /books/NBK*
238
239User-agent: Exabot
240Disallow: /Structure
241Disallow: /mailman
242Disallow: /stat
243Disallow: /sars-cov-2
244
245
246User-agent: bingbot
247Disallow: /cgi-bin
248Disallow: /entrez
249Disallow: /stat
250Disallow: /COG
251Disallow: /Entrez
252Disallow: /myncbi
253Disallow: /mailman
254Disallow: /sutils
255Disallow: /Taxonomy/Selector
256Disallow: /Taxonomy/CommonTree
257Disallow: /entrez/sutils
258Disallow: /mapview
259Disallow: /SAGE
260Disallow: /SNP
261Disallow: /snp
262Disallow: /blast/BlastAlign.cgi
263Disallow: /blast/bl2seq/wblast2.cgi
264Disallow: /portal
265Disallow: /pmc/utilities/
266Disallow: /pmc/issues/
267Disallow: /pmc/*/bin/
268Disallow: /pmc/*/pdf/
269Disallow: /pmc/*/equ/
270Disallow: /pmc/*.cgi
271Disallow: /pmc/*report=
272Disallow: /pmc/*page=
273Disallow: /pmc/articles/*/citedby/
274Disallow: /pmc/publisherportal/api/
275Disallow: /pmc/publisherportal/application/
276Disallow: /pmc/publisherportal/download
277Disallow: /pmc/publisherportal/journalmanager/
278Disallow: /labs/pmc/
279Disallow: /geo/tools
280Disallow: /geo/browse
281Disallow: /projects/geo/tools
282Disallow: /geo/download
283Disallow: /projects/geo/download
284Disallow: /projects/gtr/gt
285Disallow: /Structure/flink/flink.cgi
286Disallow: /Structure/vast/vastsrv.cgi
287Disallow: /viewvc
288Disallow: /clone/library/genomic/organisms
289Disallow: /sars-cov-2
290Allow: /portal/portal3rc.fcgi/*/*.js
291Allow: /portal/portal3rc.fcgi/*/*.css
292
293User-agent: *
294Crawl-delay: 5
295Disallow: /cgi-bin
296Disallow: /stat
297Disallow: /entrez
298Disallow: /COG
299Disallow: /Entrez
300Disallow: /myncbi
301Disallow: /mailman
302Disallow: /sutils
303Disallow: /Taxonomy/Selector
304Disallow: /Taxonomy/CommonTree
305Disallow: /entrez/sutils
306Disallow: /mapview
307Disallow: /SAGE
308Disallow: /SNP
309Disallow: /snp
310Disallow: /blast/BlastAlign.cgi
311Disallow: /blast/bl2seq/wblast2.cgi
312Disallow: /portal
313Disallow: /pmc/utilities/
314Disallow: /pmc/issues/
315Disallow: /pmc/articles/
316Disallow: /pmc/ivip/
317Disallow: /pmc/articlerender.fcgi
318Disallow: /pmc/pagerender.fcgi
319Disallow: /pmc/picrender.fcgi
320Disallow: /pmc/tocrender.fcgi
321Disallow: /pmc/*report=
322Disallow: /pmc/*page=
323Disallow: /pmc/articles/*/citedby/
324Disallow: /pmc/publisherportal/api/
325Disallow: /pmc/publisherportal/application/
326Disallow: /pmc/publisherportal/download
327Disallow: /pmc/publisherportal/journalmanager/
328Disallow: /labs/pmc/
329Disallow: /geo/tools
330Disallow: /geo/browse
331Disallow: /projects/geo/tools
332Disallow: /geo/download
333Disallow: /projects/geo/download
334Disallow: /projects/gtr/gt
335Disallow: /Structure/flink/flink.cgi
336Disallow: /Structure/vast/vastsrv.cgi
337Disallow: /viewvc
338Disallow: /clone/library/genomic/organisms
339Disallow: /sars-cov-2
340Disallow: /search/?term=
341Disallow: /books/?term=
342Disallow: /nuccore/?term=
343Disallow: /clinvar/?term=
344Disallow: /cdd/?term=
345Disallow: /gap/?term=
346Disallow: /dbvar/?term=
347Disallow: /medgen/?term=
348Disallow: /snp/?term=
349Disallow: /sra/?term=
350Disallow: /biocollections/?term=
351Disallow: /gtr/?term=
352Disallow: /protein/?term=
353Disallow: /bioproject/?term=
354Disallow: /gene/?term=
355Disallow: /biosample/?term=
356Disallow: /taxonomy/?term=
357Disallow: /ipg/?term=
358Disallow: /gds/?term=
359Disallow: /geoprofiles/?term=
360Disallow: /mesh/?term=
361Disallow: /nlmcatalog/?term=
362Disallow: /omim/?term=
363Disallow: /proteinclusters/?term=
364Disallow: /pcassay/?term=
365Disallow: /protfam/?term=
366Disallow: /pccompound/?term=
367Disallow: /pcsubstance/?term=
368Disallow: /structure/?term=
369Allow: /portal/portal3rc.fcgi/*/*.js
370Allow: /portal/portal3rc.fcgi/*/*.css
371
372Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/cdd/sitemap_index.xml
373Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/genome/sitemap_index.xml
374Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/mmdb/sitemap_index.xml
375Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/sparcle/sitemap.xml
376Sitemap: https://www.ncbi.nlm.nih.gov/Traces/sitemap_index.xml
377Sitemap: https://www.ncbi.nlm.nih.gov/core/assets/osiris/files/sitemap.xml
378Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=assembly/sitemap_index.xml
379Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=biosystems/sitemap_index.xml
380Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=books/sitemap_index.xml
381Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=cdd/sitemap_index.xml
382Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=clinvar/sitemap_index.xml
383Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=gene/sitemap_index.xml
384Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=gtr/sitemap_index.xml
385Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=medgen/sitemap_index.xml
386Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=mesh/sitemap_index.xml
387Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=ncbi/sitemap.xml
388Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=nlmcatalog/sitemap_index.xml
389Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=orgtrack/sitemap_index.xml
390Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=snp/sitemap_index.xml
391Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=sra/sitemap_index.xml