NeuralCrawl

Bloomberg / robots.txt

← back to bloomberg.com · change detected 1mo ago (2026-09-23T16:51:28Z) · modified +168 −42

What changed

Diff old (2026-07-08T20:50:19Z) → new (2026-09-23T16:51:28Z)

@@ -72,20 +72,6 @@
7272  Disallow: /
7373 
7474  # AI/LLM Bots
75- User-agent: ChatGPT-User
76- Disallow: /
77- Allow: /professional
78- Allow: /company
79- Allow: /latam
80- Allow: /faq
81- Allow: /tc
82- User-agent: GPTBot
83- Disallow: /
84- Allow: /professional
85- Allow: /company
86- Allow: /latam
87- Allow: /faq
88- Allow: /tc
8975  User-agent: Claude-Web
9076  Disallow: /
9177  Allow: /professional
@@ -114,34 +100,6 @@
114100  Allow: /latam
115101  Allow: /faq
116102  Allow: /tc
117- User-agent: ClaudeBot
118- Disallow: /
119- Allow: /professional
120- Allow: /company
121- Allow: /latam
122- Allow: /faq
123- Allow: /tc
124- User-agent: Claude-User
125- Disallow: /
126- Allow: /professional
127- Allow: /company
128- Allow: /latam
129- Allow: /faq
130- Allow: /tc
131- User-agent: Claude-SearchBot
132- Disallow: /
133- Allow: /professional
134- Allow: /company
135- Allow: /latam
136- Allow: /faq
137- Allow: /tc
138- User-agent: OAI-SearchBot
139- Disallow: /
140- Allow: /professional
141- Allow: /company
142- Allow: /latam
143- Allow: /faq
144- Allow: /tc
145103  User-agent: DuckAssistBot
146104  Disallow: /
147105  Allow: /professional
@@ -151,6 +109,174 @@
151109  Allow: /tc
152110  User-agent: cohere-ai
153111  Disallow: /
112+
113+ # AI/LLM Bots - Homepage Access
114+ # Homepage and regional homepages only. The $ makes each Allow an exact match
115+ # so it beats "Disallow: /"; without it the rule is a prefix match and opens
116+ # everything under that route. /region/ is a 302 to /region and needs its own
117+ # rule, since robots.txt is checked before the redirect can be followed.
118+
119+ # Training
120+ User-agent: GPTBot
121+ Disallow: /
122+ Allow: /professional
123+ Allow: /company
124+ Allow: /latam
125+ Allow: /faq
126+ Allow: /tc
127+ Allow: /$
128+ Allow: /europe$
129+ Allow: /europe/$
130+ Allow: /asia$
131+ Allow: /asia/$
132+ Allow: /jp$
133+ Allow: /jp/$
134+ Allow: /india$
135+ Allow: /india/$
136+ Allow: /middleeast$
137+ Allow: /middleeast/$
138+ Allow: /africa$
139+ Allow: /africa/$
140+ Allow: /canada$
141+ Allow: /canada/$
142+ Allow: /uk$
143+ Allow: /uk/$
144+ Allow: /latinamerica$
145+ Allow: /latinamerica/$
146+ User-agent: ClaudeBot
147+ Disallow: /
148+ Allow: /professional
149+ Allow: /company
150+ Allow: /latam
151+ Allow: /faq
152+ Allow: /tc
153+ Allow: /$
154+ Allow: /europe$
155+ Allow: /europe/$
156+ Allow: /asia$
157+ Allow: /asia/$
158+ Allow: /jp$
159+ Allow: /jp/$
160+ Allow: /india$
161+ Allow: /india/$
162+ Allow: /middleeast$
163+ Allow: /middleeast/$
164+ Allow: /africa$
165+ Allow: /africa/$
166+ Allow: /canada$
167+ Allow: /canada/$
168+ Allow: /uk$
169+ Allow: /uk/$
170+ Allow: /latinamerica$
171+ Allow: /latinamerica/$
172+
173+ # Search indexing
174+ User-agent: OAI-SearchBot
175+ Disallow: /
176+ Allow: /professional
177+ Allow: /company
178+ Allow: /latam
179+ Allow: /faq
180+ Allow: /tc
181+ Allow: /$
182+ Allow: /europe$
183+ Allow: /europe/$
184+ Allow: /asia$
185+ Allow: /asia/$
186+ Allow: /jp$
187+ Allow: /jp/$
188+ Allow: /india$
189+ Allow: /india/$
190+ Allow: /middleeast$
191+ Allow: /middleeast/$
192+ Allow: /africa$
193+ Allow: /africa/$
194+ Allow: /canada$
195+ Allow: /canada/$
196+ Allow: /uk$
197+ Allow: /uk/$
198+ Allow: /latinamerica$
199+ Allow: /latinamerica/$
200+ User-agent: Claude-SearchBot
201+ Disallow: /
202+ Allow: /professional
203+ Allow: /company
204+ Allow: /latam
205+ Allow: /faq
206+ Allow: /tc
207+ Allow: /$
208+ Allow: /europe$
209+ Allow: /europe/$
210+ Allow: /asia$
211+ Allow: /asia/$
212+ Allow: /jp$
213+ Allow: /jp/$
214+ Allow: /india$
215+ Allow: /india/$
216+ Allow: /middleeast$
217+ Allow: /middleeast/$
218+ Allow: /africa$
219+ Allow: /africa/$
220+ Allow: /canada$
221+ Allow: /canada/$
222+ Allow: /uk$
223+ Allow: /uk/$
224+ Allow: /latinamerica$
225+ Allow: /latinamerica/$
226+
227+ # User-triggered fetches
228+ User-agent: ChatGPT-User
229+ Disallow: /
230+ Allow: /professional
231+ Allow: /company
232+ Allow: /latam
233+ Allow: /faq
234+ Allow: /tc
235+ Allow: /$
236+ Allow: /europe$
237+ Allow: /europe/$
238+ Allow: /asia$
239+ Allow: /asia/$
240+ Allow: /jp$
241+ Allow: /jp/$
242+ Allow: /india$
243+ Allow: /india/$
244+ Allow: /middleeast$
245+ Allow: /middleeast/$
246+ Allow: /africa$
247+ Allow: /africa/$
248+ Allow: /canada$
249+ Allow: /canada/$
250+ Allow: /uk$
251+ Allow: /uk/$
252+ Allow: /latinamerica$
253+ Allow: /latinamerica/$
254+ User-agent: Claude-User
255+ Disallow: /
256+ Allow: /professional
257+ Allow: /company
258+ Allow: /latam
259+ Allow: /faq
260+ Allow: /tc
261+ Allow: /$
262+ Allow: /europe$
263+ Allow: /europe/$
264+ Allow: /asia$
265+ Allow: /asia/$
266+ Allow: /jp$
267+ Allow: /jp/$
268+ Allow: /india$
269+ Allow: /india/$
270+ Allow: /middleeast$
271+ Allow: /middleeast/$
272+ Allow: /africa$
273+ Allow: /africa/$
274+ Allow: /canada$
275+ Allow: /canada/$
276+ Allow: /uk$
277+ Allow: /uk/$
278+ Allow: /latinamerica$
279+ Allow: /latinamerica/$
154280 
155281  # Google Services
156282  User-agent: Google-Apps