arXiv / robots.txt snapshot

← back to arxiv.org · fetched 2026-06-26T14:15:22Z (4h ago) · HTTP 200 · 5854 bytes · sha256 eda2bc852443f3f2 · raw

final URL: https://arxiv.org/robots.txt

1	# robots.txt for http://arxiv.org/ and mirror sites http://*.arxiv.org/
2	# Indiscriminate automated downloads from this site are not permitted
3	# See also: http://arxiv.org/help/robots
4
5	User-agent: *
6	Crawl-delay: 15
7	Allow: /archive
8	Allow: /year
9	Allow: /list
10	Allow: /abs
11	Allow: /pdf
12	Allow: /html
13	Allow: /catchup
14	Disallow: /user
15	Disallow: /e-print
16	Disallow: /src
17	Disallow: /ps
18	Disallow: /dvi
19	Disallow: /cookies
20	Disallow: /form
21	Disallow: /find
22	Disallow: /view
23	Disallow: /ftp
24	Disallow: /refs
25	Disallow: /cits
26	Disallow: /format
27	Disallow: /PS_cache
28	Disallow: /Stats
29	Disallow: /seek-and-destroy
30	Disallow: /IgnoreMe
31	Disallow: /oai2
32	Disallow: /auth
33	Disallow: /tb
34	Disallow: /tb-recent
35	Disallow: /trackback
36	Disallow: /prevnext
37	Disallow: /ct
38	Disallow: /api
39	Disallow: /search
40	Disallow: /set_author_id
41	Disallow: /show-email
42
43	User-agent: Googlebot
44	Allow: /archive
45	Allow: /year
46	Allow: /list
47	Allow: /abs
48	Allow: /pdf
49	Allow: /html
50	Allow: /catchup
51	Disallow: /user
52	Disallow: /e-print
53	Disallow: /src
54	Disallow: /ps
55	Disallow: /dvi
56	Disallow: /cookies
57	Disallow: /form
58	Disallow: /find
59	Disallow: /view
60	Disallow: /ftp
61	Disallow: /refs
62	Disallow: /cits
63	Disallow: /format
64	Disallow: /PS_cache
65	Disallow: /Stats
66	Disallow: /seek-and-destroy
67	Disallow: /IgnoreMe
68	Disallow: /oai2
69	Disallow: /auth
70	Disallow: /tb
71	Disallow: /tb-recent
72	Disallow: /trackback
73	Disallow: /prevnext
74	Disallow: /ct
75	Disallow: /api
76	Disallow: /search
77
78	User-agent: Yahoo! Slurp
79	Crawl-delay: 1
80	Allow: /archive
81	Allow: /year
82	Allow: /list
83	Allow: /abs
84	Allow: /pdf
85	Allow: /html
86	Allow: /catchup
87	Disallow: /user
88	Disallow: /e-print
89	Disallow: /src
90	Disallow: /ps
91	Disallow: /dvi
92	Disallow: /cookies
93	Disallow: /form
94	Disallow: /find
95	Disallow: /view
96	Disallow: /ftp
97	Disallow: /refs
98	Disallow: /cits
99	Disallow: /format
100	Disallow: /PS_cache
101	Disallow: /Stats
102	Disallow: /seek-and-destroy
103	Disallow: /IgnoreMe
104	Disallow: /oai2
105	Disallow: /auth
106	Disallow: /tb
107	Disallow: /tb-recent
108	Disallow: /trackback
109	Disallow: /prevnext
110	Disallow: /ct
111	Disallow: /api
112	Disallow: /search
113
114	User-agent: bingbot
115	# 2021-10-14 - removed crawl-delay for Bingbot. Needs to be re-added if there are any problems.
116	# 2021-10-26 - added back
117	Crawl-delay: 1
118	Allow: /archive
119	Allow: /year
120	Allow: /list
121	Allow: /abs
122	Allow: /pdf
123	Allow: /html
124	Allow: /catchup
125	Disallow: /user
126	Disallow: /e-print
127	Disallow: /src
128	Disallow: /ps
129	Disallow: /dvi
130	Disallow: /cookies
131	Disallow: /form
132	Disallow: /find
133	Disallow: /view
134	Disallow: /ftp
135	Disallow: /refs
136	Disallow: /cits
137	Disallow: /format
138	Disallow: /PS_cache
139	Disallow: /Stats
140	Disallow: /seek-and-destroy
141	Disallow: /IgnoreMe
142	Disallow: /oai2
143	Disallow: /auth
144	Disallow: /tb
145	Disallow: /tb-recent
146	Disallow: /trackback
147	Disallow: /prevnext
148	Disallow: /ct
149	Disallow: /api
150	Disallow: /search
151
152	User-agent: Baiduspider
153	Crawl-delay: 10
154	Allow: /archive
155	Allow: /year
156	Allow: /list
157	Allow: /abs
158	Allow: /pdf
159	Allow: /html
160	Allow: /catchup
161	Disallow: /user
162	Disallow: /e-print
163	Disallow: /src
164	Disallow: /ps
165	Disallow: /dvi
166	Disallow: /cookies
167	Disallow: /form
168	Disallow: /find
169	Disallow: /view
170	Disallow: /ftp
171	Disallow: /refs
172	Disallow: /cits
173	Disallow: /format
174	Disallow: /PS_cache
175	Disallow: /Stats
176	Disallow: /seek-and-destroy
177	Disallow: /IgnoreMe
178	Disallow: /oai2
179	Disallow: /auth
180	Disallow: /tb
181	Disallow: /tb-recent
182	Disallow: /trackback
183	Disallow: /prevnext
184	Disallow: /ct
185	Disallow: /api
186	Disallow: /search
187
188	User-agent: ToutiaoSpider
189	Crawl-delay: 10
190	Allow: /archive
191	Allow: /year
192	Allow: /list
193	Allow: /abs
194	Allow: /pdf
195	Allow: /html
196	Allow: /catchup
197	Disallow: /user
198	Disallow: /e-print
199	Disallow: /src
200	Disallow: /ps
201	Disallow: /dvi
202	Disallow: /cookies
203	Disallow: /form
204	Disallow: /find
205	Disallow: /view
206	Disallow: /ftp
207	Disallow: /refs
208	Disallow: /cits
209	Disallow: /format
210	Disallow: /PS_cache
211	Disallow: /Stats
212	Disallow: /seek-and-destroy
213	Disallow: /IgnoreMe
214	Disallow: /oai2
215	Disallow: /auth
216	Disallow: /tb
217	Disallow: /tb-recent
218	Disallow: /trackback
219	Disallow: /prevnext
220	Disallow: /ct
221	Disallow: /api
222	Disallow: /search
223
224	User-agent: SQUID_configured_as_described_at_/help/faq/cache
225	Crawl-delay: 10
226	Allow: /list
227	Allow: /abs
228	Allow: /pdf
229	Disallow: /archive
230	Disallow: /year
231	Disallow: /html
232	Disallow: /catchup
233	Disallow: /user
234	Disallow: /e-print
235	Disallow: /src
236	Disallow: /ps
237	Disallow: /dvi
238	Disallow: /cookies
239	Disallow: /form
240	Disallow: /find
241	Disallow: /view
242	Disallow: /ftp
243	Disallow: /refs
244	Disallow: /cits
245	Disallow: /format
246	Disallow: /PS_cache
247	Disallow: /Stats
248	Disallow: /seek-and-destroy
249	Disallow: /IgnoreMe
250	Disallow: /oai2
251	Disallow: /auth
252	Disallow: /tb
253	Disallow: /tb-recent
254	Disallow: /trackback
255	Disallow: /prevnext
256	Disallow: /ct
257	Disallow: /api
258	Disallow: /search
259
260	User-agent: YandexBot
261	Crawl-delay: 1
262	Allow: /archive
263	Allow: /year
264	Allow: /list
265	Allow: /abs
266	Allow: /pdf
267	Allow: /html
268	Allow: /catchup
269	Disallow: /e-print/
270	Disallow: /src/
271	Disallow: /ps/
272	Disallow: /psfigs/
273	Disallow: /dvi/
274	Disallow: /cookies/
275	Disallow: /form/
276	Disallow: /find/
277	Disallow: /view/
278	Disallow: /ftp/
279	Disallow: /refs/
280	Disallow: /cits/
281	Disallow: /format/
282	Disallow: /register
283	Disallow: /submit
284	Disallow: /replace
285	Disallow: /cross
286	Disallow: /jref
287	Disallow: /paper_passwd/
288	Disallow: /PS_cache/
289	Disallow: /Stats/
290	Disallow: /seek-and-destroy
291	Disallow: /IgnoreMe
292	Disallow: /uploads
293	Disallow: /oai2
294	Disallow: /auth
295	Disallow: /tb
296	Disallow: /tb-recent
297	Disallow: /trackback
298	Disallow: /prevnext
299	Disallow: /ct
300	Disallow: /api
301	Disallow: /search
302
303	User-agent: Applebot
304	Crawl-delay: 1
305	Allow: /archive
306	Allow: /year
307	Allow: /list
308	Allow: /abs
309	Allow: /pdf
310	Allow: /html
311	Allow: /catchup
312	Disallow: /user
313	Disallow: /e-print
314	Disallow: /src
315	Disallow: /ps
316	Disallow: /dvi
317	Disallow: /cookies
318	Disallow: /form
319	Disallow: /find
320	Disallow: /view
321	Disallow: /ftp
322	Disallow: /refs
323	Disallow: /cits
324	Disallow: /format
325	Disallow: /PS_cache
326	Disallow: /Stats
327	Disallow: /seek-and-destroy
328	Disallow: /IgnoreMe
329	Disallow: /oai2
330	Disallow: /auth
331	Disallow: /tb
332	Disallow: /tb/recent
333	Disallow: /tb-recent
334	Disallow: /trackback
335	Disallow: /prevnext
336	Disallow: /ct
337	Disallow: /api
338	Disallow: /search
339
340	User-agent: SemrushBot
341	Disallow: /