NeuralCrawl

The Economist / robots.txt snapshot

← back to economist.com · fetched 2026-07-20T19:11:56Z (16d ago) · HTTP 200 · 2288 bytes · sha256 c170a2eaf0933e81 · raw

final URL: https://www.economist.com/robots.txt

1# robots.txt
2
3# Sitemaps
4
5Sitemap: https://www.economist.com/sitemap.xml
6Sitemap: https://www.economist.com/googlenews.xml
7
8# ==========================================
9# USER-FACING LLM BOTS
10# Allowed to fetch on behalf of a user
11# ==========================================
12
13User-agent: ChatGPT-User
14Allow: /pro$
15Allow: /pro/
16Disallow: /
17
18User-agent: Claude-SearchBot
19Allow: /pro$
20Allow: /pro/
21Disallow: /
22
23User-agent: Claude-User
24Allow: /pro$
25Allow: /pro/
26Disallow: /
27
28User-agent: Claude-Web
29Allow: /pro$
30Allow: /pro/
31Disallow: /
32
33User-agent: PerplexityBot
34Allow: /pro$
35Allow: /pro/
36Disallow: /
37
38User-agent: Perplexity-User
39Allow: /pro$
40Allow: /pro/
41Disallow: /
42
43# I have added OAI-SearchBot (OpenAI's live search agent)
44# as it behaves the same way as ChatGPT-User.
45User-agent: OAI-SearchBot
46Allow: /pro$
47Allow: /pro/
48Disallow: /
49
50# ==========================================
51# TRAINING BOTS & MASS SCRAPERS
52# Completely blocked from the entire site
53# ==========================================
54
55User-agent: GPTBot
56Disallow: /
57
58# Google-Extended is used by the Gemini app for searches on
59# behalf of the user but it also used for training
60User-agent: Google-Extended
61Disallow: /
62
63User-agent: anthropic-ai
64Disallow: /
65
66User-agent: Applebot-Extended
67Disallow: /
68
69User-Agent: Bytespider
70Disallow: /
71
72User-agent: CCBot
73Disallow: /
74
75User-agent: ClaudeBot
76Disallow: /
77
78# ==========================================
79# OTHER MISCELLANEOUS BOTS
80# Completely blocked
81# ==========================================
82
83User-agent: PiplBot
84Disallow: /
85
86User-agent: TurnitinBot
87Disallow: /
88
89User-agent: PetalBot
90Disallow: /
91
92User-agent: MoodleBot
93Disallow: /
94
95User-agent: magpie-crawler
96Disallow: /
97
98User-agent: ia_archiver
99Disallow: /
100
101User-Agent: Mozilla/5.0 (compatible; parse.ly scraper/0.16; +http://parsely.com)
102Disallow: /checkout
103
104# ==========================================
105# GENERAL CRAWLERS (e.g., Googlebot, Bingbot)
106# Allowed by default, but blocked from select paths
107# ==========================================
108User-agent: *
109Disallow: /5605/
110Disallow: /pubads.g.doubleclick.net/
111Disallow: /assets/infographic/
112Disallow: /search/
113Disallow: /search?q=
114Disallow: /graphql
115Disallow: /subscribe/getstarted/
116Disallow: /checkout
117Disallow: /switch*
118Disallow: /audio-edition-podcast/*/index.xml