Axios / robots.txt snapshot

← back to axios.com · fetched 2026-07-07T20:35:12Z (29d ago) · HTTP 200 · 645 bytes · sha256 9dbf5ea2259b2cb8 · raw

final URL: https://www.axios.com/robots.txt

1
2	# Disallow Rules
3
4	User-agent: Twitterbot/*
5	Disallow:
6	Crawl-delay: 1
7
8	User-agent: Google-Extended
9	Allow: /
10	Disallow: /core/*
11	Disallow: /admin/
12	Disallow: /login/
13	Disallow: /sso/
14	Disallow: /api/
15
16	User-agent: *
17	Disallow: /core/*
18	Disallow: /admin/
19	Disallow: /login/
20	Disallow: /sso/
21	Disallow: /api/
22
23	# Disallow Rules (bot-specific)
24
25	User-agent: CCBot
26	Disallow: /
27
28	User-agent: Bytespider
29	Disallow: /
30
31	User-agent: Diffbot
32	Disallow: /
33
34	User-agent: FacebookBot
35	Disallow: /
36
37	User-agent: ImagesiftBot
38	Disallow: /
39
40	User-agent: Scrapy
41	Disallow: /
42
43	User-agent: Amazonbot
44	Disallow: /
45
46	# Sitemaps
47
48	Sitemap: https://www.axios.com/sitemap.xml
49
50	# 8fkux4mqab196bbs