NeuralCrawl

The Economist / robots.txt snapshot

← back to economist.com · fetched 2026-08-20T18:05:43Z (3d ago) · HTTP 200 · 2650 bytes · sha256 d4669509e95e5a45 · raw

final URL: https://www.economist.com/robots.txt

1# robots.txt
2
3# Sitemaps
4
5Sitemap: https://www.economist.com/sitemap.xml
6Sitemap: https://www.economist.com/googlenews.xml
7
8# ==========================================
9# USER-FACING LLM BOTS
10# Allowed to fetch on behalf of a user
11# ==========================================
12
13User-agent: ChatGPT-User
14Allow: /pro$
15Allow: /pro/
16Allow: /syndication$
17Allow: /syndication/
18Disallow: /
19
20User-agent: Claude-SearchBot
21Allow: /pro$
22Allow: /pro/
23Allow: /syndication$
24Allow: /syndication/
25Disallow: /
26
27User-agent: Claude-User
28Allow: /pro$
29Allow: /pro/
30Allow: /syndication$
31Allow: /syndication/
32Disallow: /
33
34User-agent: Claude-Web
35Allow: /pro$
36Allow: /pro/
37Allow: /syndication$
38Allow: /syndication/
39Disallow: /
40
41User-agent: PerplexityBot
42Allow: /pro$
43Allow: /pro/
44Allow: /syndication$
45Allow: /syndication/
46Disallow: /
47
48User-agent: Perplexity-User
49Allow: /pro$
50Allow: /pro/
51Allow: /syndication$
52Allow: /syndication/
53Disallow: /
54
55# I have added OAI-SearchBot (OpenAI's live search agent)
56# as it behaves the same way as ChatGPT-User.
57User-agent: OAI-SearchBot
58Allow: /pro$
59Allow: /pro/
60Allow: /syndication$
61Allow: /syndication/
62Disallow: /
63
64# ==========================================
65# TRAINING BOTS & MASS SCRAPERS
66# Completely blocked from the entire site
67# ==========================================
68
69User-agent: GPTBot
70Disallow: /
71
72# Google-Extended is used by the Gemini app for searches on
73# behalf of the user but it also used for training
74User-agent: Google-Extended
75Allow: /pro$
76Allow: /pro/
77Allow: /syndication$
78Allow: /syndication/
79Disallow: /
80
81User-agent: anthropic-ai
82Disallow: /
83
84User-agent: Applebot-Extended
85Disallow: /
86
87User-Agent: Bytespider
88Disallow: /
89
90User-agent: CCBot
91Disallow: /
92
93User-agent: ClaudeBot
94Disallow: /
95
96# ==========================================
97# OTHER MISCELLANEOUS BOTS
98# Completely blocked
99# ==========================================
100
101User-agent: PiplBot
102Disallow: /
103
104User-agent: TurnitinBot
105Disallow: /
106
107User-agent: PetalBot
108Disallow: /
109
110User-agent: MoodleBot
111Disallow: /
112
113User-agent: magpie-crawler
114Disallow: /
115
116User-agent: ia_archiver
117Disallow: /
118
119User-Agent: Mozilla/5.0 (compatible; parse.ly scraper/0.16; +http://parsely.com)
120Disallow: /checkout
121
122# ==========================================
123# GENERAL CRAWLERS (e.g., Googlebot, Bingbot)
124# Allowed by default, but blocked from select paths
125# ==========================================
126User-agent: *
127Disallow: /5605/
128Disallow: /pubads.g.doubleclick.net/
129Disallow: /assets/infographic/
130Disallow: /search/
131Disallow: /search?q=
132Disallow: /graphql
133Disallow: /subscribe/getstarted/
134Disallow: /checkout
135Disallow: /switch*
136Disallow: /audio-edition-podcast/*/index.xml