arXiv / robots.txt snapshot
← back to arxiv.org · fetched 2026-06-26T14:15:22Z (4h ago) · HTTP 200 · 5854 bytes · sha256 eda2bc852443f3f2 · raw
final URL: https://arxiv.org/robots.txt
| 1 | # robots.txt for http://arxiv.org/ and mirror sites http://*.arxiv.org/ |
| 2 | # Indiscriminate automated downloads from this site are not permitted |
| 3 | # See also: http://arxiv.org/help/robots |
| 4 | |
| 5 | User-agent: * |
| 6 | Crawl-delay: 15 |
| 7 | Allow: /archive |
| 8 | Allow: /year |
| 9 | Allow: /list |
| 10 | Allow: /abs |
| 11 | Allow: /pdf |
| 12 | Allow: /html |
| 13 | Allow: /catchup |
| 14 | Disallow: /user |
| 15 | Disallow: /e-print |
| 16 | Disallow: /src |
| 17 | Disallow: /ps |
| 18 | Disallow: /dvi |
| 19 | Disallow: /cookies |
| 20 | Disallow: /form |
| 21 | Disallow: /find |
| 22 | Disallow: /view |
| 23 | Disallow: /ftp |
| 24 | Disallow: /refs |
| 25 | Disallow: /cits |
| 26 | Disallow: /format |
| 27 | Disallow: /PS_cache |
| 28 | Disallow: /Stats |
| 29 | Disallow: /seek-and-destroy |
| 30 | Disallow: /IgnoreMe |
| 31 | Disallow: /oai2 |
| 32 | Disallow: /auth |
| 33 | Disallow: /tb |
| 34 | Disallow: /tb-recent |
| 35 | Disallow: /trackback |
| 36 | Disallow: /prevnext |
| 37 | Disallow: /ct |
| 38 | Disallow: /api |
| 39 | Disallow: /search |
| 40 | Disallow: /set_author_id |
| 41 | Disallow: /show-email |
| 42 | |
| 43 | User-agent: Googlebot |
| 44 | Allow: /archive |
| 45 | Allow: /year |
| 46 | Allow: /list |
| 47 | Allow: /abs |
| 48 | Allow: /pdf |
| 49 | Allow: /html |
| 50 | Allow: /catchup |
| 51 | Disallow: /user |
| 52 | Disallow: /e-print |
| 53 | Disallow: /src |
| 54 | Disallow: /ps |
| 55 | Disallow: /dvi |
| 56 | Disallow: /cookies |
| 57 | Disallow: /form |
| 58 | Disallow: /find |
| 59 | Disallow: /view |
| 60 | Disallow: /ftp |
| 61 | Disallow: /refs |
| 62 | Disallow: /cits |
| 63 | Disallow: /format |
| 64 | Disallow: /PS_cache |
| 65 | Disallow: /Stats |
| 66 | Disallow: /seek-and-destroy |
| 67 | Disallow: /IgnoreMe |
| 68 | Disallow: /oai2 |
| 69 | Disallow: /auth |
| 70 | Disallow: /tb |
| 71 | Disallow: /tb-recent |
| 72 | Disallow: /trackback |
| 73 | Disallow: /prevnext |
| 74 | Disallow: /ct |
| 75 | Disallow: /api |
| 76 | Disallow: /search |
| 77 | |
| 78 | User-agent: Yahoo! Slurp |
| 79 | Crawl-delay: 1 |
| 80 | Allow: /archive |
| 81 | Allow: /year |
| 82 | Allow: /list |
| 83 | Allow: /abs |
| 84 | Allow: /pdf |
| 85 | Allow: /html |
| 86 | Allow: /catchup |
| 87 | Disallow: /user |
| 88 | Disallow: /e-print |
| 89 | Disallow: /src |
| 90 | Disallow: /ps |
| 91 | Disallow: /dvi |
| 92 | Disallow: /cookies |
| 93 | Disallow: /form |
| 94 | Disallow: /find |
| 95 | Disallow: /view |
| 96 | Disallow: /ftp |
| 97 | Disallow: /refs |
| 98 | Disallow: /cits |
| 99 | Disallow: /format |
| 100 | Disallow: /PS_cache |
| 101 | Disallow: /Stats |
| 102 | Disallow: /seek-and-destroy |
| 103 | Disallow: /IgnoreMe |
| 104 | Disallow: /oai2 |
| 105 | Disallow: /auth |
| 106 | Disallow: /tb |
| 107 | Disallow: /tb-recent |
| 108 | Disallow: /trackback |
| 109 | Disallow: /prevnext |
| 110 | Disallow: /ct |
| 111 | Disallow: /api |
| 112 | Disallow: /search |
| 113 | |
| 114 | User-agent: bingbot |
| 115 | # 2021-10-14 - removed crawl-delay for Bingbot. Needs to be re-added if there are any problems. |
| 116 | # 2021-10-26 - added back |
| 117 | Crawl-delay: 1 |
| 118 | Allow: /archive |
| 119 | Allow: /year |
| 120 | Allow: /list |
| 121 | Allow: /abs |
| 122 | Allow: /pdf |
| 123 | Allow: /html |
| 124 | Allow: /catchup |
| 125 | Disallow: /user |
| 126 | Disallow: /e-print |
| 127 | Disallow: /src |
| 128 | Disallow: /ps |
| 129 | Disallow: /dvi |
| 130 | Disallow: /cookies |
| 131 | Disallow: /form |
| 132 | Disallow: /find |
| 133 | Disallow: /view |
| 134 | Disallow: /ftp |
| 135 | Disallow: /refs |
| 136 | Disallow: /cits |
| 137 | Disallow: /format |
| 138 | Disallow: /PS_cache |
| 139 | Disallow: /Stats |
| 140 | Disallow: /seek-and-destroy |
| 141 | Disallow: /IgnoreMe |
| 142 | Disallow: /oai2 |
| 143 | Disallow: /auth |
| 144 | Disallow: /tb |
| 145 | Disallow: /tb-recent |
| 146 | Disallow: /trackback |
| 147 | Disallow: /prevnext |
| 148 | Disallow: /ct |
| 149 | Disallow: /api |
| 150 | Disallow: /search |
| 151 | |
| 152 | User-agent: Baiduspider |
| 153 | Crawl-delay: 10 |
| 154 | Allow: /archive |
| 155 | Allow: /year |
| 156 | Allow: /list |
| 157 | Allow: /abs |
| 158 | Allow: /pdf |
| 159 | Allow: /html |
| 160 | Allow: /catchup |
| 161 | Disallow: /user |
| 162 | Disallow: /e-print |
| 163 | Disallow: /src |
| 164 | Disallow: /ps |
| 165 | Disallow: /dvi |
| 166 | Disallow: /cookies |
| 167 | Disallow: /form |
| 168 | Disallow: /find |
| 169 | Disallow: /view |
| 170 | Disallow: /ftp |
| 171 | Disallow: /refs |
| 172 | Disallow: /cits |
| 173 | Disallow: /format |
| 174 | Disallow: /PS_cache |
| 175 | Disallow: /Stats |
| 176 | Disallow: /seek-and-destroy |
| 177 | Disallow: /IgnoreMe |
| 178 | Disallow: /oai2 |
| 179 | Disallow: /auth |
| 180 | Disallow: /tb |
| 181 | Disallow: /tb-recent |
| 182 | Disallow: /trackback |
| 183 | Disallow: /prevnext |
| 184 | Disallow: /ct |
| 185 | Disallow: /api |
| 186 | Disallow: /search |
| 187 | |
| 188 | User-agent: ToutiaoSpider |
| 189 | Crawl-delay: 10 |
| 190 | Allow: /archive |
| 191 | Allow: /year |
| 192 | Allow: /list |
| 193 | Allow: /abs |
| 194 | Allow: /pdf |
| 195 | Allow: /html |
| 196 | Allow: /catchup |
| 197 | Disallow: /user |
| 198 | Disallow: /e-print |
| 199 | Disallow: /src |
| 200 | Disallow: /ps |
| 201 | Disallow: /dvi |
| 202 | Disallow: /cookies |
| 203 | Disallow: /form |
| 204 | Disallow: /find |
| 205 | Disallow: /view |
| 206 | Disallow: /ftp |
| 207 | Disallow: /refs |
| 208 | Disallow: /cits |
| 209 | Disallow: /format |
| 210 | Disallow: /PS_cache |
| 211 | Disallow: /Stats |
| 212 | Disallow: /seek-and-destroy |
| 213 | Disallow: /IgnoreMe |
| 214 | Disallow: /oai2 |
| 215 | Disallow: /auth |
| 216 | Disallow: /tb |
| 217 | Disallow: /tb-recent |
| 218 | Disallow: /trackback |
| 219 | Disallow: /prevnext |
| 220 | Disallow: /ct |
| 221 | Disallow: /api |
| 222 | Disallow: /search |
| 223 | |
| 224 | User-agent: SQUID_configured_as_described_at_/help/faq/cache |
| 225 | Crawl-delay: 10 |
| 226 | Allow: /list |
| 227 | Allow: /abs |
| 228 | Allow: /pdf |
| 229 | Disallow: /archive |
| 230 | Disallow: /year |
| 231 | Disallow: /html |
| 232 | Disallow: /catchup |
| 233 | Disallow: /user |
| 234 | Disallow: /e-print |
| 235 | Disallow: /src |
| 236 | Disallow: /ps |
| 237 | Disallow: /dvi |
| 238 | Disallow: /cookies |
| 239 | Disallow: /form |
| 240 | Disallow: /find |
| 241 | Disallow: /view |
| 242 | Disallow: /ftp |
| 243 | Disallow: /refs |
| 244 | Disallow: /cits |
| 245 | Disallow: /format |
| 246 | Disallow: /PS_cache |
| 247 | Disallow: /Stats |
| 248 | Disallow: /seek-and-destroy |
| 249 | Disallow: /IgnoreMe |
| 250 | Disallow: /oai2 |
| 251 | Disallow: /auth |
| 252 | Disallow: /tb |
| 253 | Disallow: /tb-recent |
| 254 | Disallow: /trackback |
| 255 | Disallow: /prevnext |
| 256 | Disallow: /ct |
| 257 | Disallow: /api |
| 258 | Disallow: /search |
| 259 | |
| 260 | User-agent: YandexBot |
| 261 | Crawl-delay: 1 |
| 262 | Allow: /archive |
| 263 | Allow: /year |
| 264 | Allow: /list |
| 265 | Allow: /abs |
| 266 | Allow: /pdf |
| 267 | Allow: /html |
| 268 | Allow: /catchup |
| 269 | Disallow: /e-print/ |
| 270 | Disallow: /src/ |
| 271 | Disallow: /ps/ |
| 272 | Disallow: /psfigs/ |
| 273 | Disallow: /dvi/ |
| 274 | Disallow: /cookies/ |
| 275 | Disallow: /form/ |
| 276 | Disallow: /find/ |
| 277 | Disallow: /view/ |
| 278 | Disallow: /ftp/ |
| 279 | Disallow: /refs/ |
| 280 | Disallow: /cits/ |
| 281 | Disallow: /format/ |
| 282 | Disallow: /register |
| 283 | Disallow: /submit |
| 284 | Disallow: /replace |
| 285 | Disallow: /cross |
| 286 | Disallow: /jref |
| 287 | Disallow: /paper_passwd/ |
| 288 | Disallow: /PS_cache/ |
| 289 | Disallow: /Stats/ |
| 290 | Disallow: /seek-and-destroy |
| 291 | Disallow: /IgnoreMe |
| 292 | Disallow: /uploads |
| 293 | Disallow: /oai2 |
| 294 | Disallow: /auth |
| 295 | Disallow: /tb |
| 296 | Disallow: /tb-recent |
| 297 | Disallow: /trackback |
| 298 | Disallow: /prevnext |
| 299 | Disallow: /ct |
| 300 | Disallow: /api |
| 301 | Disallow: /search |
| 302 | |
| 303 | User-agent: Applebot |
| 304 | Crawl-delay: 1 |
| 305 | Allow: /archive |
| 306 | Allow: /year |
| 307 | Allow: /list |
| 308 | Allow: /abs |
| 309 | Allow: /pdf |
| 310 | Allow: /html |
| 311 | Allow: /catchup |
| 312 | Disallow: /user |
| 313 | Disallow: /e-print |
| 314 | Disallow: /src |
| 315 | Disallow: /ps |
| 316 | Disallow: /dvi |
| 317 | Disallow: /cookies |
| 318 | Disallow: /form |
| 319 | Disallow: /find |
| 320 | Disallow: /view |
| 321 | Disallow: /ftp |
| 322 | Disallow: /refs |
| 323 | Disallow: /cits |
| 324 | Disallow: /format |
| 325 | Disallow: /PS_cache |
| 326 | Disallow: /Stats |
| 327 | Disallow: /seek-and-destroy |
| 328 | Disallow: /IgnoreMe |
| 329 | Disallow: /oai2 |
| 330 | Disallow: /auth |
| 331 | Disallow: /tb |
| 332 | Disallow: /tb/recent |
| 333 | Disallow: /tb-recent |
| 334 | Disallow: /trackback |
| 335 | Disallow: /prevnext |
| 336 | Disallow: /ct |
| 337 | Disallow: /api |
| 338 | Disallow: /search |
| 339 | |
| 340 | User-agent: SemrushBot |
| 341 | Disallow: / |