NeuralCrawl

Bloomberg / robots.txt snapshot

← back to bloomberg.com · fetched 2026-10-06T03:20:59Z (1mo ago) · HTTP 200 · 10640 bytes · sha256 6879b4e3afbc243a · raw

final URL: https://www.bloomberg.com/robots.txt

1# Bot rules:
2# 1. A bot may not injure a human being or, through inaction, allow a human being to come to harm.
3# 2. A bot must obey orders given it by human beings except where such orders would conflict with the First Law.
4# 3. A bot must protect its own existence as long as such protection does not conflict with the First or Second Law.
5# If you can read this then you should apply here https://www.bloomberg.com/careers/
6User-agent: *
7Disallow: /polska
8Allow: /account/newsletters
9Disallow: /account/*
10Disallow: /tosv*.html
11Disallow: /search
12Disallow: /company/search/
13Disallow: /professional/search/
14Disallow: /impact/search/
15Disallow: /ux/search/
16Disallow: /wnwi/search/
17Disallow: /gei/search/
18Disallow: /impact/search/
19Disallow: /netzeropathfinders/search/
20Disallow: /notices/search/
21Disallow: /distribution/search/
22Disallow: /ukinnovators/search/
23Disallow: /latam/search/
24Disallow: /faq/search/
25Disallow: /tc/search/
26Disallow: /subscriptions/group/manage/
27Disallow: /preview/lineup
28Disallow: /preview/articles
29Disallow: /explore/
30Disallow: /press-releases/
31Disallow: /artemis/
32Disallow: /sessions-publisher/
33
34User-agent: Google-Extended
35Disallow: /
36Allow: /professional
37Allow: /company
38Allow: /latam
39Allow: /faq
40Allow: /tc
41
42User-agent: Mediapartners-Google
43Disallow: /about/careers
44Disallow: /about/careers/
45Disallow: /offlinemessage/
46Disallow: /apps/fbk
47Disallow: /bb/newsarchive/
48Disallow: /apps/news
49
50User-agent: Spinn3r
51Disallow: /podcasts/
52Disallow: /feed/podcast/
53Disallow: /bb/avfile/
54
55User-agent: Googlebot-News
56Disallow: /sponsor/
57Disallow: /news/sponsors/*
58Disallow: /news/terminal/*
59
60User-agent: Twitterbot
61Allow: /en/news/thp
62
63User-agent: CCBot
64Disallow: /
65
66# Development Tools
67User-agent: Python-urllib
68Disallow: /
69User-agent: python-requests
70Disallow: /
71User-agent: Python-http-client
72Disallow: /
73
74# AI/LLM Bots
75User-agent: Claude-Web
76Disallow: /
77Allow: /professional
78Allow: /company
79Allow: /latam
80Allow: /faq
81Allow: /tc
82User-agent: anthropic-ai
83Disallow: /
84Allow: /professional
85Allow: /company
86Allow: /latam
87Allow: /faq
88Allow: /tc
89User-agent: PerplexityBot
90Disallow: /
91Allow: /professional
92Allow: /company
93Allow: /latam
94Allow: /faq
95Allow: /tc
96User-agent: YouBot
97Disallow: /
98Allow: /professional
99Allow: /company
100Allow: /latam
101Allow: /faq
102Allow: /tc
103User-agent: DuckAssistBot
104Disallow: /
105Allow: /professional
106Allow: /company
107Allow: /latam
108Allow: /faq
109Allow: /tc
110User-agent: cohere-ai
111Disallow: /
112
113# AI/LLM Bots - Homepage Access
114# Homepage and regional homepages only. The $ makes each Allow an exact match
115# so it beats "Disallow: /"; without it the rule is a prefix match and opens
116# everything under that route. /region/ is a 302 to /region and needs its own
117# rule, since robots.txt is checked before the redirect can be followed.
118
119# Training
120User-agent: GPTBot
121Disallow: /
122Allow: /professional
123Allow: /company
124Allow: /latam
125Allow: /faq
126Allow: /tc
127User-agent: ClaudeBot
128Disallow: /
129Allow: /professional
130Allow: /company
131Allow: /latam
132Allow: /faq
133Allow: /tc
134
135# Search indexing
136User-agent: OAI-SearchBot
137Disallow: /
138Allow: /professional
139Allow: /company
140Allow: /latam
141Allow: /faq
142Allow: /tc
143Allow: /$
144Allow: /europe$
145Allow: /europe/$
146Allow: /asia$
147Allow: /asia/$
148Allow: /jp$
149Allow: /jp/$
150Allow: /india$
151Allow: /india/$
152Allow: /middleeast$
153Allow: /middleeast/$
154Allow: /africa$
155Allow: /africa/$
156Allow: /canada$
157Allow: /canada/$
158Allow: /uk$
159Allow: /uk/$
160Allow: /latinamerica$
161Allow: /latinamerica/$
162Allow: /llms.txt$
163Allow: /llms.txt/$
164Allow: /europe/llms.txt$
165Allow: /europe/llms.txt/$
166Allow: /asia/llms.txt$
167Allow: /asia/llms.txt/$
168Allow: /jp/llms.txt$
169Allow: /jp/llms.txt/$
170Allow: /india/llms.txt$
171Allow: /india/llms.txt/$
172Allow: /middleeast/llms.txt$
173Allow: /middleeast/llms.txt/$
174Allow: /africa/llms.txt$
175Allow: /africa/llms.txt/$
176Allow: /canada/llms.txt$
177Allow: /canada/llms.txt/$
178Allow: /uk/llms.txt$
179Allow: /uk/llms.txt/$
180Allow: /latinamerica/llms.txt$
181Allow: /latinamerica/llms.txt/$
182User-agent: Claude-SearchBot
183Disallow: /
184Allow: /professional
185Allow: /company
186Allow: /latam
187Allow: /faq
188Allow: /tc
189Allow: /$
190Allow: /europe$
191Allow: /europe/$
192Allow: /asia$
193Allow: /asia/$
194Allow: /jp$
195Allow: /jp/$
196Allow: /india$
197Allow: /india/$
198Allow: /middleeast$
199Allow: /middleeast/$
200Allow: /africa$
201Allow: /africa/$
202Allow: /canada$
203Allow: /canada/$
204Allow: /uk$
205Allow: /uk/$
206Allow: /latinamerica$
207Allow: /latinamerica/$
208Allow: /llms.txt$
209Allow: /llms.txt/$
210Allow: /europe/llms.txt$
211Allow: /europe/llms.txt/$
212Allow: /asia/llms.txt$
213Allow: /asia/llms.txt/$
214Allow: /jp/llms.txt$
215Allow: /jp/llms.txt/$
216Allow: /india/llms.txt$
217Allow: /india/llms.txt/$
218Allow: /middleeast/llms.txt$
219Allow: /middleeast/llms.txt/$
220Allow: /africa/llms.txt$
221Allow: /africa/llms.txt/$
222Allow: /canada/llms.txt$
223Allow: /canada/llms.txt/$
224Allow: /uk/llms.txt$
225Allow: /uk/llms.txt/$
226Allow: /latinamerica/llms.txt$
227Allow: /latinamerica/llms.txt/$
228
229# User-triggered fetches
230User-agent: ChatGPT-User
231Disallow: /
232Allow: /professional
233Allow: /company
234Allow: /latam
235Allow: /faq
236Allow: /tc
237Allow: /$
238Allow: /europe$
239Allow: /europe/$
240Allow: /asia$
241Allow: /asia/$
242Allow: /jp$
243Allow: /jp/$
244Allow: /india$
245Allow: /india/$
246Allow: /middleeast$
247Allow: /middleeast/$
248Allow: /africa$
249Allow: /africa/$
250Allow: /canada$
251Allow: /canada/$
252Allow: /uk$
253Allow: /uk/$
254Allow: /latinamerica$
255Allow: /latinamerica/$
256Allow: /llms.txt$
257Allow: /llms.txt/$
258Allow: /europe/llms.txt$
259Allow: /europe/llms.txt/$
260Allow: /asia/llms.txt$
261Allow: /asia/llms.txt/$
262Allow: /jp/llms.txt$
263Allow: /jp/llms.txt/$
264Allow: /india/llms.txt$
265Allow: /india/llms.txt/$
266Allow: /middleeast/llms.txt$
267Allow: /middleeast/llms.txt/$
268Allow: /africa/llms.txt$
269Allow: /africa/llms.txt/$
270Allow: /canada/llms.txt$
271Allow: /canada/llms.txt/$
272Allow: /uk/llms.txt$
273Allow: /uk/llms.txt/$
274Allow: /latinamerica/llms.txt$
275Allow: /latinamerica/llms.txt/$
276User-agent: Claude-User
277Disallow: /
278Allow: /professional
279Allow: /company
280Allow: /latam
281Allow: /faq
282Allow: /tc
283Allow: /$
284Allow: /europe$
285Allow: /europe/$
286Allow: /asia$
287Allow: /asia/$
288Allow: /jp$
289Allow: /jp/$
290Allow: /india$
291Allow: /india/$
292Allow: /middleeast$
293Allow: /middleeast/$
294Allow: /africa$
295Allow: /africa/$
296Allow: /canada$
297Allow: /canada/$
298Allow: /uk$
299Allow: /uk/$
300Allow: /latinamerica$
301Allow: /latinamerica/$
302Allow: /llms.txt$
303Allow: /llms.txt/$
304Allow: /europe/llms.txt$
305Allow: /europe/llms.txt/$
306Allow: /asia/llms.txt$
307Allow: /asia/llms.txt/$
308Allow: /jp/llms.txt$
309Allow: /jp/llms.txt/$
310Allow: /india/llms.txt$
311Allow: /india/llms.txt/$
312Allow: /middleeast/llms.txt$
313Allow: /middleeast/llms.txt/$
314Allow: /africa/llms.txt$
315Allow: /africa/llms.txt/$
316Allow: /canada/llms.txt$
317Allow: /canada/llms.txt/$
318Allow: /uk/llms.txt$
319Allow: /uk/llms.txt/$
320Allow: /latinamerica/llms.txt$
321Allow: /latinamerica/llms.txt/$
322
323# Google Services
324User-agent: Google-Apps
325Disallow: /
326User-agent: Google-Apps-Script
327Disallow: /
328User-agent: AppEngine-Google
329Disallow: /
330User-agent: Google-Cloud
331Disallow: /
332User-agent: Google-CloudVertexBot
333Disallow: /
334
335# Major Company Bots
336User-agent: AmazonBot
337Disallow: /
338User-agent: AmazonAdBot
339Disallow: /
340User-agent: ByteSpider
341Disallow: /
342User-agent: Meta-ExternalAgent
343Disallow: /
344User-agent: Meta-ExternalFetcher
345Disallow: /
346User-agent: Meta-ExternalAgent-Image
347Disallow: /
348User-agent: Applebot-Extended
349Disallow: /
350User-agent: Bytespider-Image
351Disallow: /
352
353# Feed & Content Aggregators
354User-agent: Feedly
355Disallow: /
356Allow: /professional
357Allow: /company
358Allow: /latam
359Allow: /faq
360Allow: /tc
361User-agent: FeedlyBot
362Disallow: /
363Allow: /professional
364Allow: /company
365Allow: /latam
366Allow: /faq
367Allow: /tc
368User-agent: FeedlyApp
369Disallow: /
370Allow: /professional
371Allow: /company
372Allow: /latam
373Allow: /faq
374Allow: /tc
375User-agent: MWFeedParser
376Disallow: /
377Allow: /professional
378Allow: /company
379Allow: /latam
380Allow: /faq
381Allow: /tc
382
383# Analytics & Marketing
384User-agent: comscore
385Disallow: /
386User-agent: Comscore
387Disallow: /
388User-agent: HubSpot
389Disallow: /
390User-agent: hubspot
391Disallow: /
392User-agent: Criteo
393Disallow: /
394User-agent: criteo-bot
395Disallow: /
396User-agent: Peer39_Crawler
397Disallow: /
398User-agent: rogerbot
399Disallow: /
400User-agent: linkdexbot
401Disallow: /
402User-agent: NTENT
403Disallow: /
404User-agent: AndersPinkBot
405Disallow: /
406User-agent: IndeedBot
407Disallow: /
408
409# Search Engine Bots
410User-agent: MJ12bot
411Disallow: /
412User-agent: PetalBot
413Disallow: /
414User-agent: YisouSpider
415Disallow: /
416User-agent: 360Spider
417Disallow: /
418User-agent: Qwantify
419Disallow: /
420User-agent: ToutiaoSpider
421Disallow: /
422
423# Monitoring & Security
424User-agent: Screaming Frog SEO Spider
425Disallow: /
426Allow: /professional
427Allow: /company
428Allow: /latam
429Allow: /faq
430Allow: /tc
431User-agent: PRTG
432Disallow: /
433User-agent: FreshpingBot
434Disallow: /
435User-agent: Panopta
436Disallow: /
437User-agent: DatadogSynthetics
438Disallow: /
439User-agent: Rackspace
440Disallow: /
441User-agent: censys
442Disallow: /
443User-agent: burp
444Disallow: /
445User-agent: Burp
446Disallow: /
447User-agent: check_http
448Disallow: /
449User-agent: DotcomMonitor
450Disallow: /
451User-agent: WatchSumo
452Disallow: /
453User-agent: WormlyBot
454Disallow: /
455User-agent: CalibreBot
456Disallow: /
457User-agent: AudistoBot
458Disallow: /
459
460# Social & Content
461User-agent: Hatena
462Disallow: /
463User-agent: Hatena-Bookmark
464Disallow: /
465User-agent: EveryoneSocialBot
466Disallow: /
467User-agent: Diffbot
468Disallow: /
469User-agent: CCBot
470Disallow: /
471User-agent: Netvibes
472Disallow: /
473User-agent: WebCEO
474Disallow: /
475User-agent: Postano
476Disallow: /
477User-agent: RebelMouse
478Disallow: /
479User-agent: Muck-Rack
480Disallow: /
481User-agent: InstapaperViewer
482Disallow: /
483User-agent: Twurly
484Disallow: /
485User-agent: LivelapBot
486Disallow: /
487User-agent: DatagnionBot
488Disallow: /
489User-agent: Linespider
490Disallow: /
491User-agent: Discourse
492Disallow: /
493
494# Potentially Malicious
495User-agent: Medusa
496Disallow: /
497User-agent: pingback
498Disallow: /
499User-agent: WordPress
500Disallow: /
501User-agent: wp_ping
502Disallow: /
503User-agent: MauiBot
504Disallow: /
505User-agent: ltx71
506Disallow: /
507User-agent: WeSEE
508Disallow: /
509User-agent: halebot
510Disallow: /
511User-agent: BrightBot
512Disallow: /
513
514# Gaming/3D
515User-agent: UnityPlayer
516Disallow: /
517
518# Sitemaps app
519Sitemap: https://www.bloomberg.com/sitemaps/news/index.xml
520Sitemap: https://www.bloomberg.com/sitemaps/news/latest.xml
521Sitemap: https://www.bloomberg.com/sitemaps/collections/index.xml
522Sitemap: https://www.bloomberg.com/sitemaps/media/video/index.xml
523Sitemap: https://www.bloomberg.com/sitemaps/media/audio/index.xml
524Sitemap: https://www.bloomberg.com/sitemaps/people/profiles/index.xml
525Sitemap: https://www.bloomberg.com/sitemaps/companies/public-company/index.xml
526Sitemap: https://www.bloomberg.com/sitemaps/companies/private-company/index.xml
527Sitemap: https://www.bloomberg.com/sitemaps/securites/index.xml
528
529# Billionaires, owned by graphics
530Sitemap: https://www.bloomberg.com/billionaires/sitemap.xml