NeuralCrawl

Bloomberg / robots.txt snapshot

← back to bloomberg.com · fetched 2026-07-08T20:50:19Z (28d ago) · HTTP 200 · 7048 bytes · sha256 9e9fde22f5fb63a5 · raw

final URL: https://www.bloomberg.com/robots.txt

1# Bot rules:
2# 1. A bot may not injure a human being or, through inaction, allow a human being to come to harm.
3# 2. A bot must obey orders given it by human beings except where such orders would conflict with the First Law.
4# 3. A bot must protect its own existence as long as such protection does not conflict with the First or Second Law.
5# If you can read this then you should apply here https://www.bloomberg.com/careers/
6User-agent: *
7Disallow: /polska
8Allow: /account/newsletters
9Disallow: /account/*
10Disallow: /tosv*.html
11Disallow: /search
12Disallow: /company/search/
13Disallow: /professional/search/
14Disallow: /impact/search/
15Disallow: /ux/search/
16Disallow: /wnwi/search/
17Disallow: /gei/search/
18Disallow: /impact/search/
19Disallow: /netzeropathfinders/search/
20Disallow: /notices/search/
21Disallow: /distribution/search/
22Disallow: /ukinnovators/search/
23Disallow: /latam/search/
24Disallow: /faq/search/
25Disallow: /tc/search/
26Disallow: /subscriptions/group/manage/
27Disallow: /preview/lineup
28Disallow: /preview/articles
29Disallow: /explore/
30Disallow: /press-releases/
31Disallow: /artemis/
32Disallow: /sessions-publisher/
33
34User-agent: Google-Extended
35Disallow: /
36Allow: /professional
37Allow: /company
38Allow: /latam
39Allow: /faq
40Allow: /tc
41
42User-agent: Mediapartners-Google
43Disallow: /about/careers
44Disallow: /about/careers/
45Disallow: /offlinemessage/
46Disallow: /apps/fbk
47Disallow: /bb/newsarchive/
48Disallow: /apps/news
49
50User-agent: Spinn3r
51Disallow: /podcasts/
52Disallow: /feed/podcast/
53Disallow: /bb/avfile/
54
55User-agent: Googlebot-News
56Disallow: /sponsor/
57Disallow: /news/sponsors/*
58Disallow: /news/terminal/*
59
60User-agent: Twitterbot
61Allow: /en/news/thp
62
63User-agent: CCBot
64Disallow: /
65
66# Development Tools
67User-agent: Python-urllib
68Disallow: /
69User-agent: python-requests
70Disallow: /
71User-agent: Python-http-client
72Disallow: /
73
74# AI/LLM Bots
75User-agent: ChatGPT-User
76Disallow: /
77Allow: /professional
78Allow: /company
79Allow: /latam
80Allow: /faq
81Allow: /tc
82User-agent: GPTBot
83Disallow: /
84Allow: /professional
85Allow: /company
86Allow: /latam
87Allow: /faq
88Allow: /tc
89User-agent: Claude-Web
90Disallow: /
91Allow: /professional
92Allow: /company
93Allow: /latam
94Allow: /faq
95Allow: /tc
96User-agent: anthropic-ai
97Disallow: /
98Allow: /professional
99Allow: /company
100Allow: /latam
101Allow: /faq
102Allow: /tc
103User-agent: PerplexityBot
104Disallow: /
105Allow: /professional
106Allow: /company
107Allow: /latam
108Allow: /faq
109Allow: /tc
110User-agent: YouBot
111Disallow: /
112Allow: /professional
113Allow: /company
114Allow: /latam
115Allow: /faq
116Allow: /tc
117User-agent: ClaudeBot
118Disallow: /
119Allow: /professional
120Allow: /company
121Allow: /latam
122Allow: /faq
123Allow: /tc
124User-agent: Claude-User
125Disallow: /
126Allow: /professional
127Allow: /company
128Allow: /latam
129Allow: /faq
130Allow: /tc
131User-agent: Claude-SearchBot
132Disallow: /
133Allow: /professional
134Allow: /company
135Allow: /latam
136Allow: /faq
137Allow: /tc
138User-agent: OAI-SearchBot
139Disallow: /
140Allow: /professional
141Allow: /company
142Allow: /latam
143Allow: /faq
144Allow: /tc
145User-agent: DuckAssistBot
146Disallow: /
147Allow: /professional
148Allow: /company
149Allow: /latam
150Allow: /faq
151Allow: /tc
152User-agent: cohere-ai
153Disallow: /
154
155# Google Services
156User-agent: Google-Apps
157Disallow: /
158User-agent: Google-Apps-Script
159Disallow: /
160User-agent: AppEngine-Google
161Disallow: /
162User-agent: Google-Cloud
163Disallow: /
164User-agent: Google-CloudVertexBot
165Disallow: /
166
167# Major Company Bots
168User-agent: AmazonBot
169Disallow: /
170User-agent: AmazonAdBot
171Disallow: /
172User-agent: ByteSpider
173Disallow: /
174User-agent: Meta-ExternalAgent
175Disallow: /
176User-agent: Meta-ExternalFetcher
177Disallow: /
178User-agent: Meta-ExternalAgent-Image
179Disallow: /
180User-agent: Applebot-Extended
181Disallow: /
182User-agent: Bytespider-Image
183Disallow: /
184
185# Feed & Content Aggregators
186User-agent: Feedly
187Disallow: /
188Allow: /professional
189Allow: /company
190Allow: /latam
191Allow: /faq
192Allow: /tc
193User-agent: FeedlyBot
194Disallow: /
195Allow: /professional
196Allow: /company
197Allow: /latam
198Allow: /faq
199Allow: /tc
200User-agent: FeedlyApp
201Disallow: /
202Allow: /professional
203Allow: /company
204Allow: /latam
205Allow: /faq
206Allow: /tc
207User-agent: MWFeedParser
208Disallow: /
209Allow: /professional
210Allow: /company
211Allow: /latam
212Allow: /faq
213Allow: /tc
214
215# Analytics & Marketing
216User-agent: comscore
217Disallow: /
218User-agent: Comscore
219Disallow: /
220User-agent: HubSpot
221Disallow: /
222User-agent: hubspot
223Disallow: /
224User-agent: Criteo
225Disallow: /
226User-agent: criteo-bot
227Disallow: /
228User-agent: Peer39_Crawler
229Disallow: /
230User-agent: rogerbot
231Disallow: /
232User-agent: linkdexbot
233Disallow: /
234User-agent: NTENT
235Disallow: /
236User-agent: AndersPinkBot
237Disallow: /
238User-agent: IndeedBot
239Disallow: /
240
241# Search Engine Bots
242User-agent: MJ12bot
243Disallow: /
244User-agent: PetalBot
245Disallow: /
246User-agent: YisouSpider
247Disallow: /
248User-agent: 360Spider
249Disallow: /
250User-agent: Qwantify
251Disallow: /
252User-agent: ToutiaoSpider
253Disallow: /
254
255# Monitoring & Security
256User-agent: Screaming Frog SEO Spider
257Disallow: /
258Allow: /professional
259Allow: /company
260Allow: /latam
261Allow: /faq
262Allow: /tc
263User-agent: PRTG
264Disallow: /
265User-agent: FreshpingBot
266Disallow: /
267User-agent: Panopta
268Disallow: /
269User-agent: DatadogSynthetics
270Disallow: /
271User-agent: Rackspace
272Disallow: /
273User-agent: censys
274Disallow: /
275User-agent: burp
276Disallow: /
277User-agent: Burp
278Disallow: /
279User-agent: check_http
280Disallow: /
281User-agent: DotcomMonitor
282Disallow: /
283User-agent: WatchSumo
284Disallow: /
285User-agent: WormlyBot
286Disallow: /
287User-agent: CalibreBot
288Disallow: /
289User-agent: AudistoBot
290Disallow: /
291
292# Social & Content
293User-agent: Hatena
294Disallow: /
295User-agent: Hatena-Bookmark
296Disallow: /
297User-agent: EveryoneSocialBot
298Disallow: /
299User-agent: Diffbot
300Disallow: /
301User-agent: CCBot
302Disallow: /
303User-agent: Netvibes
304Disallow: /
305User-agent: WebCEO
306Disallow: /
307User-agent: Postano
308Disallow: /
309User-agent: RebelMouse
310Disallow: /
311User-agent: Muck-Rack
312Disallow: /
313User-agent: InstapaperViewer
314Disallow: /
315User-agent: Twurly
316Disallow: /
317User-agent: LivelapBot
318Disallow: /
319User-agent: DatagnionBot
320Disallow: /
321User-agent: Linespider
322Disallow: /
323User-agent: Discourse
324Disallow: /
325
326# Potentially Malicious
327User-agent: Medusa
328Disallow: /
329User-agent: pingback
330Disallow: /
331User-agent: WordPress
332Disallow: /
333User-agent: wp_ping
334Disallow: /
335User-agent: MauiBot
336Disallow: /
337User-agent: ltx71
338Disallow: /
339User-agent: WeSEE
340Disallow: /
341User-agent: halebot
342Disallow: /
343User-agent: BrightBot
344Disallow: /
345
346# Gaming/3D
347User-agent: UnityPlayer
348Disallow: /
349
350# Sitemaps app
351Sitemap: https://www.bloomberg.com/sitemaps/news/index.xml
352Sitemap: https://www.bloomberg.com/sitemaps/news/latest.xml
353Sitemap: https://www.bloomberg.com/sitemaps/collections/index.xml
354Sitemap: https://www.bloomberg.com/sitemaps/media/video/index.xml
355Sitemap: https://www.bloomberg.com/sitemaps/media/audio/index.xml
356Sitemap: https://www.bloomberg.com/sitemaps/people/profiles/index.xml
357Sitemap: https://www.bloomberg.com/sitemaps/companies/public-company/index.xml
358Sitemap: https://www.bloomberg.com/sitemaps/companies/private-company/index.xml
359Sitemap: https://www.bloomberg.com/sitemaps/securites/index.xml
360
361# Billionaires, owned by graphics
362Sitemap: https://www.bloomberg.com/billionaires/sitemap.xml