The Asahi Shimbun / robots.txt snapshot

← back to asahi.com · fetched 2026-10-30T05:39:55Z (8d ago) · HTTP 200 · 2452 bytes · sha256 9f51e548f1bb2dc3 · raw

final URL: https://www.asahi.com/robots.txt

1	User-Agent: *
2	Disallow: /travel/event/search/
3	Disallow: /car/index.html
4	Disallow: /housing/index.html
5	Disallow: /english/newsfeatures.html
6	Disallow: /english/business.html
7	Disallow: /english/cooljapan.html
8	Disallow: /english/sports.html
9	Disallow: //search/results
10	Allow: /
11	Allow: /.well-known/assetlinks.json
12	Allow: /ads/
13	Allow: /corporate/
14
15
16	User-agent: Googlebot
17	Disallow: /klpuid=
18	Allow: /ads/
19	Allow: /corporate/
20
21	User-agent: CCBot
22	Disallow: /
23
24	User-agent: OAI-SearchBot
25	Disallow: /
26	Allow: /ads/
27	Allow: /corporate/
28
29	User-agent: ChatGPT-User
30	Disallow: /
31	Allow: /ads/
32	Allow: /corporate/
33
34	User-agent: GPTBot
35	Disallow: /
36	Allow: /ads/
37	Allow: /corporate/
38
39	User-agent: Google-Extended
40	Disallow: /
41	Allow: /ads/
42	Allow: /corporate/
43
44	User-agent: Google-CloudVertexBot
45	Disallow: /
46	Allow: /ads/
47	Allow: /corporate/
48
49	User-agent: ICC-Crawler
50	Disallow: /
51	Allow: /ads/
52	Allow: /corporate/
53
54	User-agent: anthropic-ai
55	Disallow: /
56	Allow: /ads/
57	Allow: /corporate/
58
59	User-agent: ClaudeBot
60	Disallow: /
61	Allow: /ads/
62	Allow: /corporate/
63
64	User-agent: Claude-Web
65	Disallow: /
66	Allow: /ads/
67	Allow: /corporate/
68
69	User-agent: Claude-SearchBot
70	Disallow: /
71	Allow: /ads/
72	Allow: /corporate/
73
74	User-agent: Claude-User
75	Disallow: /
76	Allow: /ads/
77	Allow: /corporate/
78
79	User-agent: Applebot-Extended
80	Disallow: /
81	Allow: /ads/
82	Allow: /corporate/
83
84	User-agent: cohere-ai
85	Disallow: /
86	Allow: /ads/
87	Allow: /corporate/
88
89	User-agent: Cohere-training-data-crawler
90	Disallow: /
91	Allow: /ads/
92	Allow: /corporate/
93
94	User-agent: omgili
95	Disallow: /
96	Allow: /ads/
97	Allow: /corporate/
98
99	User-agent: omgilibot
100	Disallow: /
101	Allow: /ads/
102	Allow: /corporate/
103
104	User-agent: PerplexityBot
105	Disallow: /
106
107	User-agent: Perplexity-ai
108	Disallow: /
109
110	User-agent: Perplexity-User
111	Disallow: /
112
113	User-agent: FacebookBot
114	Disallow: /
115	Allow: /ads/
116	Allow: /corporate/
117
118	User-agent: Meta-ExternalAgent
119	Disallow: /
120	Allow: /ads/
121	Allow: /corporate/
122
123	User-agent: Meta-externalfetcher
124	Disallow: /
125	Allow: /ads/
126	Allow: /corporate/
127
128	User-agent: Bytespider
129	Disallow: /
130
131	User-agent: Gensparkbot
132	Disallow: /
133
134	User-agent: AmazonBot
135	Disallow: /
136	Allow: /ads/
137	Allow: /corporate/
138
139	User-agent: Diffbot
140	Disallow: /
141
142	User-agent: Magpie-crawler
143	Disallow: /
144
145	User-agent: Scrapy
146	Disallow: /
147
148	User-agent: Timpibot
149	Disallow: /
150
151	User-agent: Webzio-Extended
152	Disallow: /
153
154	User-agent: SBIntuitionsBot
155	Disallow: /
156
157	User-agent: SBIntuitions-SearchBot
158	Disallow: /
159
160	User-agent: OpenindexSpider
161	Disallow: /
162
163	User-agent: AI2Bot
164	Disallow: /
165
166	sitemap: https://www.asahi.com/sitemap.xml