NeuralCrawl

Forbes / robots.txt snapshot

← back to forbes.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 10652 bytes · sha256 cf3de6a5cc96fdb1 · raw

final URL: https://www.forbes.com/robots.txt

1User-agent: *
2Allow: /ajax/follow_info*
3Allow: /ajax/getUniregUserData*
4
5Disallow: /following/
6Disallow: /search/
7Disallow: /follow/
8Disallow: /ajax/
9Disallow: /author/following/
10Disallow: /blog/following/
11Disallow: /json/
12Disallow: /header/channels/
13Disallow: /find-more/
14Disallow: /preview/
15Disallow: *wp-admin*
16Disallow: /typeahead/
17Disallow: /pepe/
18Disallow: /media-manager/
19Disallow: /coupons/visit/*
20Disallow: /test/
21Disallow: /advisor/*zx1fa*
22Disallow: /advisor/journey/
23Disallow: /wui/
24
25User-agent: googlebot-news
26Disallow: /sites/adobe/
27Disallow: /sites/arizonacommerce/
28Disallow: /sites/bmoharrisbank/
29Disallow: /sites/businessaviation/
30Disallow: /sites/capitalonespark/
31Disallow: /sites/cdw/
32Disallow: /sites/centurylink/
33Disallow: /sites/dell/
34Disallow: /sites/emc/
35Disallow: /sites/ey/
36Disallow: /sites/fedex/
37Disallow: /sites/fidelity/
38Disallow: /sites/gyro/
39Disallow: /sites/hsbc/
40Disallow: /sites/houlihanlawrence/
41Disallow: /sites/hiscox/
42Disallow: /sites/ibm/
43Disallow: /sites/infosys/
44Disallow: /sites/mdanderson/
45Disallow: /sites/medidata/
46Disallow: /sites/microsoft/
47Disallow: /sites/microsoftdynamics/
48Disallow: /sites/netapp/
49Disallow: /sites/northwesternmutual/
50Disallow: /sites/oppenheimerfunds/
51Disallow: /sites/oracle/
52Disallow: /sites/pace/
53Disallow: /sites/raymondjames/
54Disallow: /sites/sage/
55Disallow: /sites/samsung/
56Disallow: /sites/sap/
57Disallow: /sites/sungardas/
58Disallow: /sites/taboola/
59Disallow: /sites/ups/
60Disallow: /sites/delta/
61Disallow: /sites/airbus/
62Disallow: /sites/symantec/
63Disallow: /sites/paypal/
64Disallow: /sites/olddominion/
65Disallow: /sites/salesforce/
66Disallow: /sites/gapinternational/
67Disallow: /sites/sprintbusiness/
68Disallow: /sites/transamerica/
69Disallow: /sites/constantcontact/
70Disallow: /sites/robertocoin/
71Disallow: /sites/canon/
72Disallow: /sites/samsungbusiness/
73Disallow: /sites/att/
74Disallow: /sites/lasvegas/
75Disallow: /sites/iowa/
76Disallow: /sites/teradata/
77Disallow: /sites/thehartford/
78Disallow: /sites/statoil/
79Disallow: /sites/cit/
80Disallow: /sites/deloitte/
81Disallow: /sites/huawei/
82Disallow: /sites/relateiq/
83Disallow: /sites/cftc/
84Disallow: /sites/celebritycruises/
85Disallow: /sites/universityofcalifornia/
86Disallow: /sites/coxbusiness/
87Disallow: /sites/pega/
88Disallow: /sites/athenahealth/
89Disallow: /sites/etrade/
90Disallow: /sites/comcastbusiness/
91Disallow: /sites/optum/
92Disallow: /sites/seagate/
93Disallow: /sites/rbcwealthmanagement/
94Disallow: /sites/avigilon/
95Disallow: /sites/citi/
96Disallow: /sites/castlight/
97Disallow: /sites/hyatt/
98Disallow: /sites/johnhancock/
99Disallow: /sites/koreanair/
100Disallow: /sites/merrilllynch/
101Disallow: /sites/ptc/
102Disallow: /sites/toyota/
103Disallow: /sites/unify/
104Disallow: /sites/xerox/
105Disallow: /sites/united/
106Disallow: /sites/zenithwatches/
107Disallow: /sites/jpmorganchase/
108Disallow: /sites/ramcommercial/
109Disallow: /sites/puremichigan/
110Disallow: /sites/tmfgroup/
111Disallow: /sites/usaa/
112Disallow: /sites/emerson/
113Disallow: /sites/aberdeen/
114Disallow: /sites/kellogg/
115Disallow: /sites/zurich/
116Disallow: /sites/freeenterprise/
117Disallow: /sites/allclearid/
118Disallow: /sites/accenture/
119Disallow: /sites/mercedesbenz/
120Disallow: /sites/nice/
121Disallow: /sites/riverbed/
122Disallow: /sites/pimco/
123Disallow: /sites/netjets/
124Disallow: /sites/vmware/
125Disallow: /sites/visa/
126Disallow: /sites/statefarm/
127Disallow: /sites/philips/
128Disallow: /sites/adp/
129Disallow: /sites/zenithwatches/
130Disallow: /sites/citrix/
131Disallow: /sites/gwcic/
132Disallow: /sites/cargill/
133Disallow: /sites/bankofsingapore/
134Disallow: /sites/dbsbank/
135Disallow: /sites/gradsoflife/
136Disallow: /sites/internationaliotcouncil/
137Disallow: /pega/customers-for-life
138Disallow: /celebritycruises/guide-to-the-globe
139Disallow: /sites/disneyinstitute/
140Disallow: /sites/concertocloud/
141Disallow: /sites/capitalonefutureedge/
142Disallow: /sites/mahindracomviva/
143Disallow: /sites/workday/
144Disallow: /sites/colorado/
145Disallow: /sites/charlesschwab/
146Disallow: /sites/chicagobooth/
147Disallow: /sites/morganstanley/
148Disallow: /sites/opentext/
149Disallow: /sites/icims/
150Disallow: /sites/thriventmutualfunds/
151Disallow: /sites/cbre/
152Disallow: /sites/universityofphoenix/#57934e047260
153Disallow: /sites/ppg/
154Disallow: /sites/americanfunds/
155Disallow: /sites/lilly/
156Disallow: /sites/norton/
157Disallow: /sites/qualityassurance/
158Disallow: /sites/qlik/
159Disallow: /whats-next/planes-trains-and-analytics
160Disallow: /top-50-wealth-managers
161Disallow: /toyota/fueled-by-skepticism
162Disallow: /toyota/coming-soon
163Disallow: /dell/cloud-future-of-medicine
164Disallow: /cit/renewable-energy
165Disallow: /retail
166Disallow: /mobile/cities-transformed
167Disallow: /cit/small-business-mergers-2
168Disallow: /cit/small-business-mergers
169Disallow: /sites/kpmg/
170Disallow: /dell/cities-transformed
171Disallow: /sites/neustar
172Disallow: /sites/ubs
173Disallow: /sites/northerntrust
174Disallow: /sites/infiniti
175Disallow: /sites/honeywell
176Disallow: /sites/charlesschwabadvisor
177Disallow: /sites/frisco
178Disallow: /sites/colehaan
179Disallow: /sites/gapincthiswayahead
180Disallow: /sites/discoverpersonalloans
181Disallow: /sites/delltechnologies
182Disallow: /colehaan/daring-to-step-forward
183Disallow: /johnhancock/
184Disallow: /johnhancock/in-a-lifetime-healthcare
185Disallow: /johnhancock/in-a-lifetime-work
186Disallow: /oppenheimerfunds/the-upside
187Disallow: /sites/hpeenterprisenxt
188Disallow: /sites/rackspace
189Disallow: /sites/themacallan
190Disallow: /themacallan/worlds-together
191Disallow: /sites/inteliotpartnership
192Disallow: /oppenheimerfunds/the-upside
193Disallow: /partners/
194Disallow: /johnhancock/in-a-lifetime-sustainability
195Disallow: /sites/domperignon
196Disallow: /sites/shinola
197Disallow: /sites/hpinc
198Disallow: /sites/hp
199Disallow: /sites/alfresco
200Disallow: /cbre/built-for-performance
201Disallow: /sites/mmedric
202Disallow: /sites/indiana
203Disallow: /sites/siemenshealthineers
204Disallow: /sites/rackspace
205Disallow: /rackspace/elevating-performance
206Disallow: /sites/suntrust
207Disallow: /sites/unicefusa
208Disallow: /sites/mitsubishiheavyindustries
209Disallow: /best-countries-for-business/list
210Disallow: /sites/paloaltonetworks
211Disallow: /sites/f5
212Disallow: /sites/samsungtv
213Disallow: /sites/qa
214Disallow: /sites/paycom
215Disallow: /sites/testbrandvoice
216Disallow: /business/blahblah
217Disallow: /sites/spdr
218Disallow: /sites/lincoln
219Disallow: /sites/abb
220Disallow: /sites/eycybersecurity
221Disallow: /sites/civicnation
222Disallow: /sites/bcg
223Disallow: /sites/vanguard
224Disallow: /sites/akamai
225Disallow: /sites/kyocera
226Disallow: /sites/socialimpact
227Disallow: /sites/emirates
228Disallow: /sites/hpincemea
229Disallow: /sites/ibmperformancecomputing
230Disallow: /sites/pfizer
231Disallow: /sites/ibmpowersystems
232Disallow: /sites/vanguardcharitable
233Disallow: /sites/suntrustprivatewealth
234Disallow: /sites/capitalone
235Disallow: /sites/juniper
236Disallow: /sites/wellsfargo
237Disallow: /sites/bnymellon
238Disallow: /sites/braintree
239Disallow: /sites/bnymellonwealthmanagement
240Disallow: /sites/epson
241Disallow: /sites/thomsonreuters
242Disallow: /sites/paysafe
243Disallow: /sites/josabank
244Disallow: /sites/hennessyparadisimperial
245Disallow: /sites/cartier
246Disallow: /sites/anabluewing
247Disallow: /sites/eidoo
248Disallow: /sites/cmstesting
249Disallow: /sites/adobepartnership
250Disallow: /sites/cornelljohnson
251Disallow: /sites/quantcast
252Disallow: /sites/japan
253Disallow: /sites/impactpartners
254Disallow: /sites/nvidia
255Disallow: /sites/personalcapital
256Disallow: /sites/microsoft365
257Disallow: /sites/providencedigitalinnovation
258Disallow: /sites/whittiertrust
259Disallow: /sites/kinetica
260Disallow: /sites/paulevans
261Disallow: /sites/ritz-carltonleadershipcenter
262Disallow: /sites/icoengine
263Disallow: /sites/tesaro
264Disallow: /sites/theglenlivet
265Disallow: /feature/sf-test
266Disallow: /sites/courtyardmarriott
267Disallow: /sites/nuveen
268Disallow: /sites/intelai
269Disallow: /sites/secret
270Disallow: /sites/hec
271Disallow: /retail/old
272Disallow: /sites/alliantgroup
273Disallow: /sites/intuitivesurgical
274Disallow: /sites/sunovion
275Disallow: /sites/ofx
276Disallow: /sites/firstrepublicbank
277Disallow: /sites/1850byfolgers
278Disallow: /sites/audi
279Disallow: /sites/nuveencef
280Disallow: /sites/1850brandcoffee
281Disallow: /sites/cisco
282Disallow: /sites/blackstarcapitalltd
283Disallow: /sites/usbankaltitudereserve
284Disallow: /sites/dassaultsystemes
285Disallow: /sites/urogen
286Disallow: /sites/smartsheet
287Disallow: /sites/iwc
288Disallow: /sites/datingcom
289Disallow: /sites/intuitquickbooks
290Disallow: /sites/oticon
291Disallow: /sites/servicenow
292Disallow: /sites/deloitteriskandfinancialadvisory
293Disallow: /sites/dbsprivatebank
294Disallow: /sites/americanexpress
295Disallow: /sites/tanotofoundation
296Disallow: /sites/citizensaccess
297Disallow: /sites/salesforceessentials
298Disallow: /sites/drexeluniversity
299Disallow: /sites/microsoftinnovation
300Disallow: /sites/mypos
301Disallow: /sites/harvardpilgrimhealthcare
302Disallow: /sites/appdynamics
303Disallow: /sites/ministryoftourismindia
304Disallow: /sites/orlando
305Disallow: /sites/extrahop
306Disallow: /sites/ascm
307Disallow: /sites/crowe
308Disallow: /sites/peerfit
309Disallow: /sites/theleukemialymphomasociety
310Disallow: /sites/hewlettpackardenterprise
311Disallow: /sites/montblanc
312Disallow: /sites/meero
313Disallow: /sites/braze
314Disallow: /sites/publicissapient
315Disallow: /sites/flexport
316
317User-agent: Amazonbot
318Disallow: /
319
320User-agent: anthropic-ai
321Disallow: /
322
323User-agent: Applebot-Extended
324Disallow: /
325
326User-agent: Bytespider
327Disallow: /
328
329User-agent: CCBot
330Disallow: /
331
332User-agent: ClaudeBot
333Disallow: /
334
335User-agent: cohere-ai
336Disallow: /
337
338User-agent: Diffbot
339Disallow: /
340
341User-agent: FacebookBot
342Disallow: /
343
344User-agent: FriendlyCrawler
345Disallow: /
346
347User-agent: GPTBot
348Disallow: /
349
350User-agent: ImagesiftBot
351Disallow: /
352
353User-agent: meta-externalagent
354User-agent: Meta-ExternalAgent
355Disallow: /
356
357User-Agent: OAI-SearchBot
358Allow: /
359
360User-agent: omgili
361Disallow: /
362
363User-agent: omgilibot
364Disallow: /
365
366User-Agent: PerplexityBot
367Disallow: /
368
369User-agent: AhrefsBot
370User-agent: AhrefsSiteAudit
371Crawl-delay: 2
372
373User-Agent: Claude-User
374User-Agent: Claude-SearchBot
375Allow: /
376
377User-agent: AmazonAdBot
378Allow: /following/
379Allow: /search/
380Allow: /follow/
381Allow: /author/following/
382Allow: /blog/following/
383Allow: /header/channels/
384Allow: /find-more/
385Allow: /preview/
386Allow: /typeahead/
387Allow: /pepe/
388Allow: /media-manager/
389Allow: /coupons/visit/*
390Allow: /advisor/*zx1fa*
391Allow: /wui/
392
393Sitemap: https://www.forbes.com/news_sitemap.xml
394Sitemap: https://www.forbes.com/video_sitemap.xml
395Sitemap: https://www.forbes.com/companies_sitemap.xml
396Sitemap: https://www.forbes.com/health/sitemap.xml
397Sitemap: https://www.forbes.com/connect/events-sitemap.xml
398Sitemap: https://www.forbes.com/advisor/sitemap.xml
399Sitemap: https://www.forbes.com/home-improvement/sitemap.xml
400Sitemap: https://www.forbes.com/digital-assets/sitemap.xml