NeuralCrawl

Bloomberg / robots.txt

← back to bloomberg.com · change detected 4d ago (2026-11-03T00:58:14Z) · modified +43 −255

What changed

Diff old (2026-10-06T03:20:59Z) → new (2026-11-03T00:58:14Z)

@@ -65,41 +65,15 @@
6565 
6666  # Development Tools
6767  User-agent: Python-urllib
68- Disallow: /
6968  User-agent: python-requests
70- Disallow: /
7169  User-agent: Python-http-client
7270  Disallow: /
7371 
7472  # AI/LLM Bots
7573  User-agent: Claude-Web
76- Disallow: /
77- Allow: /professional
78- Allow: /company
79- Allow: /latam
80- Allow: /faq
81- Allow: /tc
8274  User-agent: anthropic-ai
83- Disallow: /
84- Allow: /professional
85- Allow: /company
86- Allow: /latam
87- Allow: /faq
88- Allow: /tc
8975  User-agent: PerplexityBot
90- Disallow: /
91- Allow: /professional
92- Allow: /company
93- Allow: /latam
94- Allow: /faq
95- Allow: /tc
9676  User-agent: YouBot
97- Disallow: /
98- Allow: /professional
99- Allow: /company
100- Allow: /latam
101- Allow: /faq
102- Allow: /tc
10377  User-agent: DuckAssistBot
10478  Disallow: /
10579  Allow: /professional
@@ -118,12 +92,6 @@
11892 
11993  # Training
12094  User-agent: GPTBot
121- Disallow: /
122- Allow: /professional
123- Allow: /company
124- Allow: /latam
125- Allow: /faq
126- Allow: /tc
12795  User-agent: ClaudeBot
12896  Disallow: /
12997  Allow: /professional
@@ -132,147 +100,13 @@
132100  Allow: /faq
133101  Allow: /tc
134102 
135- # Search indexing
103+ # Search indexing and user-triggered fetches
104+ # Each exact Allow has a ?bloomberg_debug= twin so we can tag our own test
105+ # fetches and find them in the Fastly logs, i.e.
106+ # /llms.txt?bloomberg_debug=digest-20261102
136107  User-agent: OAI-SearchBot
137- Disallow: /
138- Allow: /professional
139- Allow: /company
140- Allow: /latam
141- Allow: /faq
142- Allow: /tc
143- Allow: /$
144- Allow: /europe$
145- Allow: /europe/$
146- Allow: /asia$
147- Allow: /asia/$
148- Allow: /jp$
149- Allow: /jp/$
150- Allow: /india$
151- Allow: /india/$
152- Allow: /middleeast$
153- Allow: /middleeast/$
154- Allow: /africa$
155- Allow: /africa/$
156- Allow: /canada$
157- Allow: /canada/$
158- Allow: /uk$
159- Allow: /uk/$
160- Allow: /latinamerica$
161- Allow: /latinamerica/$
162- Allow: /llms.txt$
163- Allow: /llms.txt/$
164- Allow: /europe/llms.txt$
165- Allow: /europe/llms.txt/$
166- Allow: /asia/llms.txt$
167- Allow: /asia/llms.txt/$
168- Allow: /jp/llms.txt$
169- Allow: /jp/llms.txt/$
170- Allow: /india/llms.txt$
171- Allow: /india/llms.txt/$
172- Allow: /middleeast/llms.txt$
173- Allow: /middleeast/llms.txt/$
174- Allow: /africa/llms.txt$
175- Allow: /africa/llms.txt/$
176- Allow: /canada/llms.txt$
177- Allow: /canada/llms.txt/$
178- Allow: /uk/llms.txt$
179- Allow: /uk/llms.txt/$
180- Allow: /latinamerica/llms.txt$
181- Allow: /latinamerica/llms.txt/$
182108  User-agent: Claude-SearchBot
183- Disallow: /
184- Allow: /professional
185- Allow: /company
186- Allow: /latam
187- Allow: /faq
188- Allow: /tc
189- Allow: /$
190- Allow: /europe$
191- Allow: /europe/$
192- Allow: /asia$
193- Allow: /asia/$
194- Allow: /jp$
195- Allow: /jp/$
196- Allow: /india$
197- Allow: /india/$
198- Allow: /middleeast$
199- Allow: /middleeast/$
200- Allow: /africa$
201- Allow: /africa/$
202- Allow: /canada$
203- Allow: /canada/$
204- Allow: /uk$
205- Allow: /uk/$
206- Allow: /latinamerica$
207- Allow: /latinamerica/$
208- Allow: /llms.txt$
209- Allow: /llms.txt/$
210- Allow: /europe/llms.txt$
211- Allow: /europe/llms.txt/$
212- Allow: /asia/llms.txt$
213- Allow: /asia/llms.txt/$
214- Allow: /jp/llms.txt$
215- Allow: /jp/llms.txt/$
216- Allow: /india/llms.txt$
217- Allow: /india/llms.txt/$
218- Allow: /middleeast/llms.txt$
219- Allow: /middleeast/llms.txt/$
220- Allow: /africa/llms.txt$
221- Allow: /africa/llms.txt/$
222- Allow: /canada/llms.txt$
223- Allow: /canada/llms.txt/$
224- Allow: /uk/llms.txt$
225- Allow: /uk/llms.txt/$
226- Allow: /latinamerica/llms.txt$
227- Allow: /latinamerica/llms.txt/$
228-
229- # User-triggered fetches
230109  User-agent: ChatGPT-User
231- Disallow: /
232- Allow: /professional
233- Allow: /company
234- Allow: /latam
235- Allow: /faq
236- Allow: /tc
237- Allow: /$
238- Allow: /europe$
239- Allow: /europe/$
240- Allow: /asia$
241- Allow: /asia/$
242- Allow: /jp$
243- Allow: /jp/$
244- Allow: /india$
245- Allow: /india/$
246- Allow: /middleeast$
247- Allow: /middleeast/$
248- Allow: /africa$
249- Allow: /africa/$
250- Allow: /canada$
251- Allow: /canada/$
252- Allow: /uk$
253- Allow: /uk/$
254- Allow: /latinamerica$
255- Allow: /latinamerica/$
256- Allow: /llms.txt$
257- Allow: /llms.txt/$
258- Allow: /europe/llms.txt$
259- Allow: /europe/llms.txt/$
260- Allow: /asia/llms.txt$
261- Allow: /asia/llms.txt/$
262- Allow: /jp/llms.txt$
263- Allow: /jp/llms.txt/$
264- Allow: /india/llms.txt$
265- Allow: /india/llms.txt/$
266- Allow: /middleeast/llms.txt$
267- Allow: /middleeast/llms.txt/$
268- Allow: /africa/llms.txt$
269- Allow: /africa/llms.txt/$
270- Allow: /canada/llms.txt$
271- Allow: /canada/llms.txt/$
272- Allow: /uk/llms.txt$
273- Allow: /uk/llms.txt/$
274- Allow: /latinamerica/llms.txt$
275- Allow: /latinamerica/llms.txt/$
276110  User-agent: Claude-User
277111  Disallow: /
278112  Allow: /professional
@@ -281,97 +115,107 @@
281115  Allow: /faq
282116  Allow: /tc
283117  Allow: /$
118+ Allow: /?bloomberg_debug=
284119  Allow: /europe$
120+ Allow: /europe?bloomberg_debug=
285121  Allow: /europe/$
122+ Allow: /europe/?bloomberg_debug=
286123  Allow: /asia$
124+ Allow: /asia?bloomberg_debug=
287125  Allow: /asia/$
126+ Allow: /asia/?bloomberg_debug=
288127  Allow: /jp$
128+ Allow: /jp?bloomberg_debug=
289129  Allow: /jp/$
130+ Allow: /jp/?bloomberg_debug=
290131  Allow: /india$
132+ Allow: /india?bloomberg_debug=
291133  Allow: /india/$
134+ Allow: /india/?bloomberg_debug=
292135  Allow: /middleeast$
136+ Allow: /middleeast?bloomberg_debug=
293137  Allow: /middleeast/$
138+ Allow: /middleeast/?bloomberg_debug=
294139  Allow: /africa$
140+ Allow: /africa?bloomberg_debug=
295141  Allow: /africa/$
142+ Allow: /africa/?bloomberg_debug=
296143  Allow: /canada$
144+ Allow: /canada?bloomberg_debug=
297145  Allow: /canada/$
146+ Allow: /canada/?bloomberg_debug=
298147  Allow: /uk$
148+ Allow: /uk?bloomberg_debug=
299149  Allow: /uk/$
150+ Allow: /uk/?bloomberg_debug=
300151  Allow: /latinamerica$
152+ Allow: /latinamerica?bloomberg_debug=
301153  Allow: /latinamerica/$
154+ Allow: /latinamerica/?bloomberg_debug=
302155  Allow: /llms.txt$
156+ Allow: /llms.txt?bloomberg_debug=
303157  Allow: /llms.txt/$
158+ Allow: /llms.txt/?bloomberg_debug=
304159  Allow: /europe/llms.txt$
160+ Allow: /europe/llms.txt?bloomberg_debug=
305161  Allow: /europe/llms.txt/$
162+ Allow: /europe/llms.txt/?bloomberg_debug=
306163  Allow: /asia/llms.txt$
164+ Allow: /asia/llms.txt?bloomberg_debug=
307165  Allow: /asia/llms.txt/$
166+ Allow: /asia/llms.txt/?bloomberg_debug=
308167  Allow: /jp/llms.txt$
168+ Allow: /jp/llms.txt?bloomberg_debug=
309169  Allow: /jp/llms.txt/$
170+ Allow: /jp/llms.txt/?bloomberg_debug=
310171  Allow: /india/llms.txt$
172+ Allow: /india/llms.txt?bloomberg_debug=
311173  Allow: /india/llms.txt/$
174+ Allow: /india/llms.txt/?bloomberg_debug=
312175  Allow: /middleeast/llms.txt$
176+ Allow: /middleeast/llms.txt?bloomberg_debug=
313177  Allow: /middleeast/llms.txt/$
178+ Allow: /middleeast/llms.txt/?bloomberg_debug=
314179  Allow: /africa/llms.txt$
180+ Allow: /africa/llms.txt?bloomberg_debug=
315181  Allow: /africa/llms.txt/$
182+ Allow: /africa/llms.txt/?bloomberg_debug=
316183  Allow: /canada/llms.txt$
184+ Allow: /canada/llms.txt?bloomberg_debug=
317185  Allow: /canada/llms.txt/$
186+ Allow: /canada/llms.txt/?bloomberg_debug=
318187  Allow: /uk/llms.txt$
188+ Allow: /uk/llms.txt?bloomberg_debug=
319189  Allow: /uk/llms.txt/$
190+ Allow: /uk/llms.txt/?bloomberg_debug=
320191  Allow: /latinamerica/llms.txt$
192+ Allow: /latinamerica/llms.txt?bloomberg_debug=
321193  Allow: /latinamerica/llms.txt/$
194+ Allow: /latinamerica/llms.txt/?bloomberg_debug=
322195 
323196  # Google Services
324197  User-agent: Google-Apps
325- Disallow: /
326198  User-agent: Google-Apps-Script
327- Disallow: /
328199  User-agent: AppEngine-Google
329- Disallow: /
330200  User-agent: Google-Cloud
331- Disallow: /
332201  User-agent: Google-CloudVertexBot
333202  Disallow: /
334203 
335204  # Major Company Bots
336205  User-agent: AmazonBot
337- Disallow: /
338206  User-agent: AmazonAdBot
339- Disallow: /
340207  User-agent: ByteSpider
341- Disallow: /
342208  User-agent: Meta-ExternalAgent
343- Disallow: /
344209  User-agent: Meta-ExternalFetcher
345- Disallow: /
346210  User-agent: Meta-ExternalAgent-Image
347- Disallow: /
348211  User-agent: Applebot-Extended
349- Disallow: /
350212  User-agent: Bytespider-Image
351213  Disallow: /
352214 
353215  # Feed & Content Aggregators
354216  User-agent: Feedly
355- Disallow: /
356- Allow: /professional
357- Allow: /company
358- Allow: /latam
359- Allow: /faq
360- Allow: /tc
361217  User-agent: FeedlyBot
362- Disallow: /
363- Allow: /professional
364- Allow: /company
365- Allow: /latam
366- Allow: /faq
367- Allow: /tc
368218  User-agent: FeedlyApp
369- Disallow: /
370- Allow: /professional
371- Allow: /company
372- Allow: /latam
373- Allow: /faq
374- Allow: /tc
375219  User-agent: MWFeedParser
376220  Disallow: /
377221  Allow: /professional
@@ -382,41 +226,23 @@
382226 
383227  # Analytics & Marketing
384228  User-agent: comscore
385- Disallow: /
386- User-agent: Comscore
387- Disallow: /
388229  User-agent: HubSpot
389- Disallow: /
390- User-agent: hubspot
391- Disallow: /
392230  User-agent: Criteo
393- Disallow: /
394231  User-agent: criteo-bot
395- Disallow: /
396232  User-agent: Peer39_Crawler
397- Disallow: /
398233  User-agent: rogerbot
399- Disallow: /
400234  User-agent: linkdexbot
401- Disallow: /
402235  User-agent: NTENT
403- Disallow: /
404236  User-agent: AndersPinkBot
405- Disallow: /
406237  User-agent: IndeedBot
407238  Disallow: /
408239 
409240  # Search Engine Bots
410241  User-agent: MJ12bot
411- Disallow: /
412242  User-agent: PetalBot
413- Disallow: /
414243  User-agent: YisouSpider
415- Disallow: /
416244  User-agent: 360Spider
417- Disallow: /
418245  User-agent: Qwantify
419- Disallow: /
420246  User-agent: ToutiaoSpider
421247  Disallow: /
422248 
@@ -429,85 +255,47 @@
429255  Allow: /faq
430256  Allow: /tc
431257  User-agent: PRTG
432- Disallow: /
433258  User-agent: FreshpingBot
434- Disallow: /
435259  User-agent: Panopta
436- Disallow: /
437260  User-agent: DatadogSynthetics
438- Disallow: /
439261  User-agent: Rackspace
440- Disallow: /
441262  User-agent: censys
442- Disallow: /
443263  User-agent: burp
444- Disallow: /
445- User-agent: Burp
446- Disallow: /
447264  User-agent: check_http
448- Disallow: /
449265  User-agent: DotcomMonitor
450- Disallow: /
451266  User-agent: WatchSumo
452- Disallow: /
453267  User-agent: WormlyBot
454- Disallow: /
455268  User-agent: CalibreBot
456- Disallow: /
457269  User-agent: AudistoBot
458270  Disallow: /
459271 
460272  # Social & Content
461273  User-agent: Hatena
462- Disallow: /
463274  User-agent: Hatena-Bookmark
464- Disallow: /
465275  User-agent: EveryoneSocialBot
466- Disallow: /
467276  User-agent: Diffbot
468- Disallow: /
469- User-agent: CCBot
470- Disallow: /
471277  User-agent: Netvibes
472- Disallow: /
473278  User-agent: WebCEO
474- Disallow: /
475279  User-agent: Postano
476- Disallow: /
477280  User-agent: RebelMouse
478- Disallow: /
479281  User-agent: Muck-Rack
480- Disallow: /
481282  User-agent: InstapaperViewer
482- Disallow: /
483283  User-agent: Twurly
484- Disallow: /
485284  User-agent: LivelapBot
486- Disallow: /
487285  User-agent: DatagnionBot
488- Disallow: /
489286  User-agent: Linespider
490- Disallow: /
491287  User-agent: Discourse
492288  Disallow: /
493289 
494290  # Potentially Malicious
495291  User-agent: Medusa
496- Disallow: /
497292  User-agent: pingback
498- Disallow: /
499293  User-agent: WordPress
500- Disallow: /
501294  User-agent: wp_ping
502- Disallow: /
503295  User-agent: MauiBot
504- Disallow: /
505296  User-agent: ltx71
506- Disallow: /
507297  User-agent: WeSEE
508- Disallow: /
509298  User-agent: halebot
510- Disallow: /
511299  User-agent: BrightBot
512300  Disallow: /
513301