NeuralCrawl

Bloomberg / robots.txt snapshot

← back to bloomberg.com · fetched 2026-09-28T18:34:49Z (1mo ago) · HTTP 200 · 8664 bytes · sha256 cf8884bf1e30d131 · raw

final URL: https://www.bloomberg.com/robots.txt

1# Bot rules:
2# 1. A bot may not injure a human being or, through inaction, allow a human being to come to harm.
3# 2. A bot must obey orders given it by human beings except where such orders would conflict with the First Law.
4# 3. A bot must protect its own existence as long as such protection does not conflict with the First or Second Law.
5# If you can read this then you should apply here https://www.bloomberg.com/careers/
6User-agent: *
7Disallow: /polska
8Allow: /account/newsletters
9Disallow: /account/*
10Disallow: /tosv*.html
11Disallow: /search
12Disallow: /company/search/
13Disallow: /professional/search/
14Disallow: /impact/search/
15Disallow: /ux/search/
16Disallow: /wnwi/search/
17Disallow: /gei/search/
18Disallow: /impact/search/
19Disallow: /netzeropathfinders/search/
20Disallow: /notices/search/
21Disallow: /distribution/search/
22Disallow: /ukinnovators/search/
23Disallow: /latam/search/
24Disallow: /faq/search/
25Disallow: /tc/search/
26Disallow: /subscriptions/group/manage/
27Disallow: /preview/lineup
28Disallow: /preview/articles
29Disallow: /explore/
30Disallow: /press-releases/
31Disallow: /artemis/
32Disallow: /sessions-publisher/
33
34User-agent: Google-Extended
35Disallow: /
36Allow: /professional
37Allow: /company
38Allow: /latam
39Allow: /faq
40Allow: /tc
41
42User-agent: Mediapartners-Google
43Disallow: /about/careers
44Disallow: /about/careers/
45Disallow: /offlinemessage/
46Disallow: /apps/fbk
47Disallow: /bb/newsarchive/
48Disallow: /apps/news
49
50User-agent: Spinn3r
51Disallow: /podcasts/
52Disallow: /feed/podcast/
53Disallow: /bb/avfile/
54
55User-agent: Googlebot-News
56Disallow: /sponsor/
57Disallow: /news/sponsors/*
58Disallow: /news/terminal/*
59
60User-agent: Twitterbot
61Allow: /en/news/thp
62
63User-agent: CCBot
64Disallow: /
65
66# Development Tools
67User-agent: Python-urllib
68Disallow: /
69User-agent: python-requests
70Disallow: /
71User-agent: Python-http-client
72Disallow: /
73
74# AI/LLM Bots
75User-agent: Claude-Web
76Disallow: /
77Allow: /professional
78Allow: /company
79Allow: /latam
80Allow: /faq
81Allow: /tc
82User-agent: anthropic-ai
83Disallow: /
84Allow: /professional
85Allow: /company
86Allow: /latam
87Allow: /faq
88Allow: /tc
89User-agent: PerplexityBot
90Disallow: /
91Allow: /professional
92Allow: /company
93Allow: /latam
94Allow: /faq
95Allow: /tc
96User-agent: YouBot
97Disallow: /
98Allow: /professional
99Allow: /company
100Allow: /latam
101Allow: /faq
102Allow: /tc
103User-agent: DuckAssistBot
104Disallow: /
105Allow: /professional
106Allow: /company
107Allow: /latam
108Allow: /faq
109Allow: /tc
110User-agent: cohere-ai
111Disallow: /
112
113# AI/LLM Bots - Homepage Access
114# Homepage and regional homepages only. The $ makes each Allow an exact match
115# so it beats "Disallow: /"; without it the rule is a prefix match and opens
116# everything under that route. /region/ is a 302 to /region and needs its own
117# rule, since robots.txt is checked before the redirect can be followed.
118
119# Training
120User-agent: GPTBot
121Disallow: /
122Allow: /professional
123Allow: /company
124Allow: /latam
125Allow: /faq
126Allow: /tc
127User-agent: ClaudeBot
128Disallow: /
129Allow: /professional
130Allow: /company
131Allow: /latam
132Allow: /faq
133Allow: /tc
134
135# Search indexing
136User-agent: OAI-SearchBot
137Disallow: /
138Allow: /professional
139Allow: /company
140Allow: /latam
141Allow: /faq
142Allow: /tc
143Allow: /$
144Allow: /europe$
145Allow: /europe/$
146Allow: /asia$
147Allow: /asia/$
148Allow: /jp$
149Allow: /jp/$
150Allow: /india$
151Allow: /india/$
152Allow: /middleeast$
153Allow: /middleeast/$
154Allow: /africa$
155Allow: /africa/$
156Allow: /canada$
157Allow: /canada/$
158Allow: /uk$
159Allow: /uk/$
160Allow: /latinamerica$
161Allow: /latinamerica/$
162User-agent: Claude-SearchBot
163Disallow: /
164Allow: /professional
165Allow: /company
166Allow: /latam
167Allow: /faq
168Allow: /tc
169Allow: /$
170Allow: /europe$
171Allow: /europe/$
172Allow: /asia$
173Allow: /asia/$
174Allow: /jp$
175Allow: /jp/$
176Allow: /india$
177Allow: /india/$
178Allow: /middleeast$
179Allow: /middleeast/$
180Allow: /africa$
181Allow: /africa/$
182Allow: /canada$
183Allow: /canada/$
184Allow: /uk$
185Allow: /uk/$
186Allow: /latinamerica$
187Allow: /latinamerica/$
188
189# User-triggered fetches
190User-agent: ChatGPT-User
191Disallow: /
192Allow: /professional
193Allow: /company
194Allow: /latam
195Allow: /faq
196Allow: /tc
197Allow: /$
198Allow: /europe$
199Allow: /europe/$
200Allow: /asia$
201Allow: /asia/$
202Allow: /jp$
203Allow: /jp/$
204Allow: /india$
205Allow: /india/$
206Allow: /middleeast$
207Allow: /middleeast/$
208Allow: /africa$
209Allow: /africa/$
210Allow: /canada$
211Allow: /canada/$
212Allow: /uk$
213Allow: /uk/$
214Allow: /latinamerica$
215Allow: /latinamerica/$
216User-agent: Claude-User
217Disallow: /
218Allow: /professional
219Allow: /company
220Allow: /latam
221Allow: /faq
222Allow: /tc
223Allow: /$
224Allow: /europe$
225Allow: /europe/$
226Allow: /asia$
227Allow: /asia/$
228Allow: /jp$
229Allow: /jp/$
230Allow: /india$
231Allow: /india/$
232Allow: /middleeast$
233Allow: /middleeast/$
234Allow: /africa$
235Allow: /africa/$
236Allow: /canada$
237Allow: /canada/$
238Allow: /uk$
239Allow: /uk/$
240Allow: /latinamerica$
241Allow: /latinamerica/$
242
243# Google Services
244User-agent: Google-Apps
245Disallow: /
246User-agent: Google-Apps-Script
247Disallow: /
248User-agent: AppEngine-Google
249Disallow: /
250User-agent: Google-Cloud
251Disallow: /
252User-agent: Google-CloudVertexBot
253Disallow: /
254
255# Major Company Bots
256User-agent: AmazonBot
257Disallow: /
258User-agent: AmazonAdBot
259Disallow: /
260User-agent: ByteSpider
261Disallow: /
262User-agent: Meta-ExternalAgent
263Disallow: /
264User-agent: Meta-ExternalFetcher
265Disallow: /
266User-agent: Meta-ExternalAgent-Image
267Disallow: /
268User-agent: Applebot-Extended
269Disallow: /
270User-agent: Bytespider-Image
271Disallow: /
272
273# Feed & Content Aggregators
274User-agent: Feedly
275Disallow: /
276Allow: /professional
277Allow: /company
278Allow: /latam
279Allow: /faq
280Allow: /tc
281User-agent: FeedlyBot
282Disallow: /
283Allow: /professional
284Allow: /company
285Allow: /latam
286Allow: /faq
287Allow: /tc
288User-agent: FeedlyApp
289Disallow: /
290Allow: /professional
291Allow: /company
292Allow: /latam
293Allow: /faq
294Allow: /tc
295User-agent: MWFeedParser
296Disallow: /
297Allow: /professional
298Allow: /company
299Allow: /latam
300Allow: /faq
301Allow: /tc
302
303# Analytics & Marketing
304User-agent: comscore
305Disallow: /
306User-agent: Comscore
307Disallow: /
308User-agent: HubSpot
309Disallow: /
310User-agent: hubspot
311Disallow: /
312User-agent: Criteo
313Disallow: /
314User-agent: criteo-bot
315Disallow: /
316User-agent: Peer39_Crawler
317Disallow: /
318User-agent: rogerbot
319Disallow: /
320User-agent: linkdexbot
321Disallow: /
322User-agent: NTENT
323Disallow: /
324User-agent: AndersPinkBot
325Disallow: /
326User-agent: IndeedBot
327Disallow: /
328
329# Search Engine Bots
330User-agent: MJ12bot
331Disallow: /
332User-agent: PetalBot
333Disallow: /
334User-agent: YisouSpider
335Disallow: /
336User-agent: 360Spider
337Disallow: /
338User-agent: Qwantify
339Disallow: /
340User-agent: ToutiaoSpider
341Disallow: /
342
343# Monitoring & Security
344User-agent: Screaming Frog SEO Spider
345Disallow: /
346Allow: /professional
347Allow: /company
348Allow: /latam
349Allow: /faq
350Allow: /tc
351User-agent: PRTG
352Disallow: /
353User-agent: FreshpingBot
354Disallow: /
355User-agent: Panopta
356Disallow: /
357User-agent: DatadogSynthetics
358Disallow: /
359User-agent: Rackspace
360Disallow: /
361User-agent: censys
362Disallow: /
363User-agent: burp
364Disallow: /
365User-agent: Burp
366Disallow: /
367User-agent: check_http
368Disallow: /
369User-agent: DotcomMonitor
370Disallow: /
371User-agent: WatchSumo
372Disallow: /
373User-agent: WormlyBot
374Disallow: /
375User-agent: CalibreBot
376Disallow: /
377User-agent: AudistoBot
378Disallow: /
379
380# Social & Content
381User-agent: Hatena
382Disallow: /
383User-agent: Hatena-Bookmark
384Disallow: /
385User-agent: EveryoneSocialBot
386Disallow: /
387User-agent: Diffbot
388Disallow: /
389User-agent: CCBot
390Disallow: /
391User-agent: Netvibes
392Disallow: /
393User-agent: WebCEO
394Disallow: /
395User-agent: Postano
396Disallow: /
397User-agent: RebelMouse
398Disallow: /
399User-agent: Muck-Rack
400Disallow: /
401User-agent: InstapaperViewer
402Disallow: /
403User-agent: Twurly
404Disallow: /
405User-agent: LivelapBot
406Disallow: /
407User-agent: DatagnionBot
408Disallow: /
409User-agent: Linespider
410Disallow: /
411User-agent: Discourse
412Disallow: /
413
414# Potentially Malicious
415User-agent: Medusa
416Disallow: /
417User-agent: pingback
418Disallow: /
419User-agent: WordPress
420Disallow: /
421User-agent: wp_ping
422Disallow: /
423User-agent: MauiBot
424Disallow: /
425User-agent: ltx71
426Disallow: /
427User-agent: WeSEE
428Disallow: /
429User-agent: halebot
430Disallow: /
431User-agent: BrightBot
432Disallow: /
433
434# Gaming/3D
435User-agent: UnityPlayer
436Disallow: /
437
438# Sitemaps app
439Sitemap: https://www.bloomberg.com/sitemaps/news/index.xml
440Sitemap: https://www.bloomberg.com/sitemaps/news/latest.xml
441Sitemap: https://www.bloomberg.com/sitemaps/collections/index.xml
442Sitemap: https://www.bloomberg.com/sitemaps/media/video/index.xml
443Sitemap: https://www.bloomberg.com/sitemaps/media/audio/index.xml
444Sitemap: https://www.bloomberg.com/sitemaps/people/profiles/index.xml
445Sitemap: https://www.bloomberg.com/sitemaps/companies/public-company/index.xml
446Sitemap: https://www.bloomberg.com/sitemaps/companies/private-company/index.xml
447Sitemap: https://www.bloomberg.com/sitemaps/securites/index.xml
448
449# Billionaires, owned by graphics
450Sitemap: https://www.bloomberg.com/billionaires/sitemap.xml