NeuralCrawl

Bloomberg / robots.txt snapshot

← back to bloomberg.com · fetched 2026-09-23T16:51:28Z (1mo ago) · HTTP 200 · 9274 bytes · sha256 43160c24093b297f · raw

final URL: https://www.bloomberg.com/robots.txt

1# Bot rules:
2# 1. A bot may not injure a human being or, through inaction, allow a human being to come to harm.
3# 2. A bot must obey orders given it by human beings except where such orders would conflict with the First Law.
4# 3. A bot must protect its own existence as long as such protection does not conflict with the First or Second Law.
5# If you can read this then you should apply here https://www.bloomberg.com/careers/
6User-agent: *
7Disallow: /polska
8Allow: /account/newsletters
9Disallow: /account/*
10Disallow: /tosv*.html
11Disallow: /search
12Disallow: /company/search/
13Disallow: /professional/search/
14Disallow: /impact/search/
15Disallow: /ux/search/
16Disallow: /wnwi/search/
17Disallow: /gei/search/
18Disallow: /impact/search/
19Disallow: /netzeropathfinders/search/
20Disallow: /notices/search/
21Disallow: /distribution/search/
22Disallow: /ukinnovators/search/
23Disallow: /latam/search/
24Disallow: /faq/search/
25Disallow: /tc/search/
26Disallow: /subscriptions/group/manage/
27Disallow: /preview/lineup
28Disallow: /preview/articles
29Disallow: /explore/
30Disallow: /press-releases/
31Disallow: /artemis/
32Disallow: /sessions-publisher/
33
34User-agent: Google-Extended
35Disallow: /
36Allow: /professional
37Allow: /company
38Allow: /latam
39Allow: /faq
40Allow: /tc
41
42User-agent: Mediapartners-Google
43Disallow: /about/careers
44Disallow: /about/careers/
45Disallow: /offlinemessage/
46Disallow: /apps/fbk
47Disallow: /bb/newsarchive/
48Disallow: /apps/news
49
50User-agent: Spinn3r
51Disallow: /podcasts/
52Disallow: /feed/podcast/
53Disallow: /bb/avfile/
54
55User-agent: Googlebot-News
56Disallow: /sponsor/
57Disallow: /news/sponsors/*
58Disallow: /news/terminal/*
59
60User-agent: Twitterbot
61Allow: /en/news/thp
62
63User-agent: CCBot
64Disallow: /
65
66# Development Tools
67User-agent: Python-urllib
68Disallow: /
69User-agent: python-requests
70Disallow: /
71User-agent: Python-http-client
72Disallow: /
73
74# AI/LLM Bots
75User-agent: Claude-Web
76Disallow: /
77Allow: /professional
78Allow: /company
79Allow: /latam
80Allow: /faq
81Allow: /tc
82User-agent: anthropic-ai
83Disallow: /
84Allow: /professional
85Allow: /company
86Allow: /latam
87Allow: /faq
88Allow: /tc
89User-agent: PerplexityBot
90Disallow: /
91Allow: /professional
92Allow: /company
93Allow: /latam
94Allow: /faq
95Allow: /tc
96User-agent: YouBot
97Disallow: /
98Allow: /professional
99Allow: /company
100Allow: /latam
101Allow: /faq
102Allow: /tc
103User-agent: DuckAssistBot
104Disallow: /
105Allow: /professional
106Allow: /company
107Allow: /latam
108Allow: /faq
109Allow: /tc
110User-agent: cohere-ai
111Disallow: /
112
113# AI/LLM Bots - Homepage Access
114# Homepage and regional homepages only. The $ makes each Allow an exact match
115# so it beats "Disallow: /"; without it the rule is a prefix match and opens
116# everything under that route. /region/ is a 302 to /region and needs its own
117# rule, since robots.txt is checked before the redirect can be followed.
118
119# Training
120User-agent: GPTBot
121Disallow: /
122Allow: /professional
123Allow: /company
124Allow: /latam
125Allow: /faq
126Allow: /tc
127Allow: /$
128Allow: /europe$
129Allow: /europe/$
130Allow: /asia$
131Allow: /asia/$
132Allow: /jp$
133Allow: /jp/$
134Allow: /india$
135Allow: /india/$
136Allow: /middleeast$
137Allow: /middleeast/$
138Allow: /africa$
139Allow: /africa/$
140Allow: /canada$
141Allow: /canada/$
142Allow: /uk$
143Allow: /uk/$
144Allow: /latinamerica$
145Allow: /latinamerica/$
146User-agent: ClaudeBot
147Disallow: /
148Allow: /professional
149Allow: /company
150Allow: /latam
151Allow: /faq
152Allow: /tc
153Allow: /$
154Allow: /europe$
155Allow: /europe/$
156Allow: /asia$
157Allow: /asia/$
158Allow: /jp$
159Allow: /jp/$
160Allow: /india$
161Allow: /india/$
162Allow: /middleeast$
163Allow: /middleeast/$
164Allow: /africa$
165Allow: /africa/$
166Allow: /canada$
167Allow: /canada/$
168Allow: /uk$
169Allow: /uk/$
170Allow: /latinamerica$
171Allow: /latinamerica/$
172
173# Search indexing
174User-agent: OAI-SearchBot
175Disallow: /
176Allow: /professional
177Allow: /company
178Allow: /latam
179Allow: /faq
180Allow: /tc
181Allow: /$
182Allow: /europe$
183Allow: /europe/$
184Allow: /asia$
185Allow: /asia/$
186Allow: /jp$
187Allow: /jp/$
188Allow: /india$
189Allow: /india/$
190Allow: /middleeast$
191Allow: /middleeast/$
192Allow: /africa$
193Allow: /africa/$
194Allow: /canada$
195Allow: /canada/$
196Allow: /uk$
197Allow: /uk/$
198Allow: /latinamerica$
199Allow: /latinamerica/$
200User-agent: Claude-SearchBot
201Disallow: /
202Allow: /professional
203Allow: /company
204Allow: /latam
205Allow: /faq
206Allow: /tc
207Allow: /$
208Allow: /europe$
209Allow: /europe/$
210Allow: /asia$
211Allow: /asia/$
212Allow: /jp$
213Allow: /jp/$
214Allow: /india$
215Allow: /india/$
216Allow: /middleeast$
217Allow: /middleeast/$
218Allow: /africa$
219Allow: /africa/$
220Allow: /canada$
221Allow: /canada/$
222Allow: /uk$
223Allow: /uk/$
224Allow: /latinamerica$
225Allow: /latinamerica/$
226
227# User-triggered fetches
228User-agent: ChatGPT-User
229Disallow: /
230Allow: /professional
231Allow: /company
232Allow: /latam
233Allow: /faq
234Allow: /tc
235Allow: /$
236Allow: /europe$
237Allow: /europe/$
238Allow: /asia$
239Allow: /asia/$
240Allow: /jp$
241Allow: /jp/$
242Allow: /india$
243Allow: /india/$
244Allow: /middleeast$
245Allow: /middleeast/$
246Allow: /africa$
247Allow: /africa/$
248Allow: /canada$
249Allow: /canada/$
250Allow: /uk$
251Allow: /uk/$
252Allow: /latinamerica$
253Allow: /latinamerica/$
254User-agent: Claude-User
255Disallow: /
256Allow: /professional
257Allow: /company
258Allow: /latam
259Allow: /faq
260Allow: /tc
261Allow: /$
262Allow: /europe$
263Allow: /europe/$
264Allow: /asia$
265Allow: /asia/$
266Allow: /jp$
267Allow: /jp/$
268Allow: /india$
269Allow: /india/$
270Allow: /middleeast$
271Allow: /middleeast/$
272Allow: /africa$
273Allow: /africa/$
274Allow: /canada$
275Allow: /canada/$
276Allow: /uk$
277Allow: /uk/$
278Allow: /latinamerica$
279Allow: /latinamerica/$
280
281# Google Services
282User-agent: Google-Apps
283Disallow: /
284User-agent: Google-Apps-Script
285Disallow: /
286User-agent: AppEngine-Google
287Disallow: /
288User-agent: Google-Cloud
289Disallow: /
290User-agent: Google-CloudVertexBot
291Disallow: /
292
293# Major Company Bots
294User-agent: AmazonBot
295Disallow: /
296User-agent: AmazonAdBot
297Disallow: /
298User-agent: ByteSpider
299Disallow: /
300User-agent: Meta-ExternalAgent
301Disallow: /
302User-agent: Meta-ExternalFetcher
303Disallow: /
304User-agent: Meta-ExternalAgent-Image
305Disallow: /
306User-agent: Applebot-Extended
307Disallow: /
308User-agent: Bytespider-Image
309Disallow: /
310
311# Feed & Content Aggregators
312User-agent: Feedly
313Disallow: /
314Allow: /professional
315Allow: /company
316Allow: /latam
317Allow: /faq
318Allow: /tc
319User-agent: FeedlyBot
320Disallow: /
321Allow: /professional
322Allow: /company
323Allow: /latam
324Allow: /faq
325Allow: /tc
326User-agent: FeedlyApp
327Disallow: /
328Allow: /professional
329Allow: /company
330Allow: /latam
331Allow: /faq
332Allow: /tc
333User-agent: MWFeedParser
334Disallow: /
335Allow: /professional
336Allow: /company
337Allow: /latam
338Allow: /faq
339Allow: /tc
340
341# Analytics & Marketing
342User-agent: comscore
343Disallow: /
344User-agent: Comscore
345Disallow: /
346User-agent: HubSpot
347Disallow: /
348User-agent: hubspot
349Disallow: /
350User-agent: Criteo
351Disallow: /
352User-agent: criteo-bot
353Disallow: /
354User-agent: Peer39_Crawler
355Disallow: /
356User-agent: rogerbot
357Disallow: /
358User-agent: linkdexbot
359Disallow: /
360User-agent: NTENT
361Disallow: /
362User-agent: AndersPinkBot
363Disallow: /
364User-agent: IndeedBot
365Disallow: /
366
367# Search Engine Bots
368User-agent: MJ12bot
369Disallow: /
370User-agent: PetalBot
371Disallow: /
372User-agent: YisouSpider
373Disallow: /
374User-agent: 360Spider
375Disallow: /
376User-agent: Qwantify
377Disallow: /
378User-agent: ToutiaoSpider
379Disallow: /
380
381# Monitoring & Security
382User-agent: Screaming Frog SEO Spider
383Disallow: /
384Allow: /professional
385Allow: /company
386Allow: /latam
387Allow: /faq
388Allow: /tc
389User-agent: PRTG
390Disallow: /
391User-agent: FreshpingBot
392Disallow: /
393User-agent: Panopta
394Disallow: /
395User-agent: DatadogSynthetics
396Disallow: /
397User-agent: Rackspace
398Disallow: /
399User-agent: censys
400Disallow: /
401User-agent: burp
402Disallow: /
403User-agent: Burp
404Disallow: /
405User-agent: check_http
406Disallow: /
407User-agent: DotcomMonitor
408Disallow: /
409User-agent: WatchSumo
410Disallow: /
411User-agent: WormlyBot
412Disallow: /
413User-agent: CalibreBot
414Disallow: /
415User-agent: AudistoBot
416Disallow: /
417
418# Social & Content
419User-agent: Hatena
420Disallow: /
421User-agent: Hatena-Bookmark
422Disallow: /
423User-agent: EveryoneSocialBot
424Disallow: /
425User-agent: Diffbot
426Disallow: /
427User-agent: CCBot
428Disallow: /
429User-agent: Netvibes
430Disallow: /
431User-agent: WebCEO
432Disallow: /
433User-agent: Postano
434Disallow: /
435User-agent: RebelMouse
436Disallow: /
437User-agent: Muck-Rack
438Disallow: /
439User-agent: InstapaperViewer
440Disallow: /
441User-agent: Twurly
442Disallow: /
443User-agent: LivelapBot
444Disallow: /
445User-agent: DatagnionBot
446Disallow: /
447User-agent: Linespider
448Disallow: /
449User-agent: Discourse
450Disallow: /
451
452# Potentially Malicious
453User-agent: Medusa
454Disallow: /
455User-agent: pingback
456Disallow: /
457User-agent: WordPress
458Disallow: /
459User-agent: wp_ping
460Disallow: /
461User-agent: MauiBot
462Disallow: /
463User-agent: ltx71
464Disallow: /
465User-agent: WeSEE
466Disallow: /
467User-agent: halebot
468Disallow: /
469User-agent: BrightBot
470Disallow: /
471
472# Gaming/3D
473User-agent: UnityPlayer
474Disallow: /
475
476# Sitemaps app
477Sitemap: https://www.bloomberg.com/sitemaps/news/index.xml
478Sitemap: https://www.bloomberg.com/sitemaps/news/latest.xml
479Sitemap: https://www.bloomberg.com/sitemaps/collections/index.xml
480Sitemap: https://www.bloomberg.com/sitemaps/media/video/index.xml
481Sitemap: https://www.bloomberg.com/sitemaps/media/audio/index.xml
482Sitemap: https://www.bloomberg.com/sitemaps/people/profiles/index.xml
483Sitemap: https://www.bloomberg.com/sitemaps/companies/public-company/index.xml
484Sitemap: https://www.bloomberg.com/sitemaps/companies/private-company/index.xml
485Sitemap: https://www.bloomberg.com/sitemaps/securites/index.xml
486
487# Billionaires, owned by graphics
488Sitemap: https://www.bloomberg.com/billionaires/sitemap.xml