NeuralCrawl

The Asahi Shimbun / robots.txt snapshot

← back to asahi.com · fetched 2026-08-04T06:35:30Z (1d ago) · HTTP 200 · 2034 bytes · sha256 59bb843d4b6cae62 · raw

final URL: https://www.asahi.com/robots.txt

1User-Agent: *
2Disallow: /travel/event/search/
3Disallow: /car/index.html
4Disallow: /housing/index.html
5Disallow: /english/newsfeatures.html
6Disallow: /english/business.html
7Disallow: /english/cooljapan.html
8Disallow: /english/sports.html
9Disallow: /*/search/results*
10Allow: /
11Allow: /.well-known/assetlinks.json
12Allow: /ads/
13
14
15User-agent: Googlebot
16Disallow: /*klpuid=*
17Allow: /ads/
18
19User-agent: CCBot
20Disallow: /
21
22User-agent: OAI-SearchBot
23Disallow: /
24Allow: /ads/
25
26User-agent: ChatGPT-User
27Disallow: /
28Allow: /ads/
29
30User-agent: GPTBot
31Disallow: /
32Allow: /ads/
33
34User-agent: Google-Extended
35Disallow: /
36Allow: /ads/
37
38User-agent: Google-CloudVertexBot
39Disallow: /
40Allow: /ads/
41
42User-agent: ICC-Crawler
43Disallow: /
44Allow: /ads/
45
46User-agent: anthropic-ai
47Disallow: /
48Allow: /ads/
49
50User-agent: ClaudeBot
51Disallow: /
52Allow: /ads/
53
54User-agent: Claude-Web
55Disallow: /
56Allow: /ads/
57
58User-agent: Claude-SearchBot
59Disallow: /
60Allow: /ads/
61
62User-agent: Claude-User
63Disallow: /
64Allow: /ads/
65
66User-agent: Applebot-Extended
67Disallow: /
68Allow: /ads/
69
70User-agent: cohere-ai
71Disallow: /
72Allow: /ads/
73
74User-agent: Cohere-training-data-crawler
75Disallow: /
76Allow: /ads/
77
78User-agent: omgili
79Disallow: /
80Allow: /ads/
81
82User-agent: omgilibot
83Disallow: /
84Allow: /ads/
85
86User-agent: PerplexityBot
87Disallow: /
88
89User-agent: Perplexity-ai
90Disallow: /
91
92User-agent: Perplexity-User
93Disallow: /
94
95User-agent: FacebookBot
96Disallow: /
97Allow: /ads/
98
99User-agent: Meta-ExternalAgent
100Disallow: /
101Allow: /ads/
102
103User-agent: Meta-externalfetcher
104Disallow: /
105Allow: /ads/
106
107User-agent: Bytespider
108Disallow: /
109
110User-agent: Gensparkbot
111Disallow: /
112
113User-agent: AmazonBot
114Disallow: /
115Allow: /ads/
116
117User-agent: Diffbot
118Disallow: /
119
120User-agent: Magpie-crawler
121Disallow: /
122
123User-agent: Scrapy
124Disallow: /
125
126User-agent: Timpibot
127Disallow: /
128
129User-agent: Webzio-Extended
130Disallow: /
131
132User-agent: SBIntuitionsBot
133Disallow: /
134
135User-agent: SBIntuitions-SearchBot
136Disallow: /
137
138User-agent: OpenindexSpider
139Disallow: /
140
141User-agent: AI2Bot
142Disallow: /
143
144sitemap: https://www.asahi.com/sitemap.xml