NeuralCrawl

Forbes / robots.txt snapshot

← back to forbes.com · fetched 2026-08-04T18:49:16Z (5h ago) · HTTP 200 · 10719 bytes · sha256 945fd312a8432c57 · raw

final URL: https://www.forbes.com/robots.txt

1User-agent: *
2Allow: /ajax/follow_info*
3Allow: /ajax/getUniregUserData*
4
5Disallow: /following/
6Disallow: /search/
7Disallow: /follow/
8Disallow: /ajax/
9Disallow: /author/following/
10Disallow: /blog/following/
11Disallow: /json/
12Disallow: /header/channels/
13Disallow: /find-more/
14Disallow: /preview/
15Disallow: *wp-admin*
16Disallow: /typeahead/
17Disallow: /pepe/
18Disallow: /media-manager/
19Disallow: /coupons/visit/*
20Disallow: /test/
21Disallow: /advisor/*zx1fa*
22Disallow: /advisor/journey/
23Disallow: /wui/
24Disallow: /research-survey
25
26User-agent: googlebot-news
27Disallow: /sites/adobe/
28Disallow: /sites/arizonacommerce/
29Disallow: /sites/bmoharrisbank/
30Disallow: /sites/businessaviation/
31Disallow: /sites/capitalonespark/
32Disallow: /sites/cdw/
33Disallow: /sites/centurylink/
34Disallow: /sites/dell/
35Disallow: /sites/emc/
36Disallow: /sites/ey/
37Disallow: /sites/fedex/
38Disallow: /sites/fidelity/
39Disallow: /sites/gyro/
40Disallow: /sites/hsbc/
41Disallow: /sites/houlihanlawrence/
42Disallow: /sites/hiscox/
43Disallow: /sites/ibm/
44Disallow: /sites/infosys/
45Disallow: /sites/mdanderson/
46Disallow: /sites/medidata/
47Disallow: /sites/microsoft/
48Disallow: /sites/microsoftdynamics/
49Disallow: /sites/netapp/
50Disallow: /sites/northwesternmutual/
51Disallow: /sites/oppenheimerfunds/
52Disallow: /sites/oracle/
53Disallow: /sites/pace/
54Disallow: /sites/raymondjames/
55Disallow: /sites/sage/
56Disallow: /sites/samsung/
57Disallow: /sites/sap/
58Disallow: /sites/sungardas/
59Disallow: /sites/taboola/
60Disallow: /sites/ups/
61Disallow: /sites/delta/
62Disallow: /sites/airbus/
63Disallow: /sites/symantec/
64Disallow: /sites/paypal/
65Disallow: /sites/olddominion/
66Disallow: /sites/salesforce/
67Disallow: /sites/gapinternational/
68Disallow: /sites/sprintbusiness/
69Disallow: /sites/transamerica/
70Disallow: /sites/constantcontact/
71Disallow: /sites/robertocoin/
72Disallow: /sites/canon/
73Disallow: /sites/samsungbusiness/
74Disallow: /sites/att/
75Disallow: /sites/lasvegas/
76Disallow: /sites/iowa/
77Disallow: /sites/teradata/
78Disallow: /sites/thehartford/
79Disallow: /sites/statoil/
80Disallow: /sites/cit/
81Disallow: /sites/deloitte/
82Disallow: /sites/huawei/
83Disallow: /sites/relateiq/
84Disallow: /sites/cftc/
85Disallow: /sites/celebritycruises/
86Disallow: /sites/universityofcalifornia/
87Disallow: /sites/coxbusiness/
88Disallow: /sites/pega/
89Disallow: /sites/athenahealth/
90Disallow: /sites/etrade/
91Disallow: /sites/comcastbusiness/
92Disallow: /sites/optum/
93Disallow: /sites/seagate/
94Disallow: /sites/rbcwealthmanagement/
95Disallow: /sites/avigilon/
96Disallow: /sites/citi/
97Disallow: /sites/castlight/
98Disallow: /sites/hyatt/
99Disallow: /sites/johnhancock/
100Disallow: /sites/koreanair/
101Disallow: /sites/merrilllynch/
102Disallow: /sites/ptc/
103Disallow: /sites/toyota/
104Disallow: /sites/unify/
105Disallow: /sites/xerox/
106Disallow: /sites/united/
107Disallow: /sites/zenithwatches/
108Disallow: /sites/jpmorganchase/
109Disallow: /sites/ramcommercial/
110Disallow: /sites/puremichigan/
111Disallow: /sites/tmfgroup/
112Disallow: /sites/usaa/
113Disallow: /sites/emerson/
114Disallow: /sites/aberdeen/
115Disallow: /sites/kellogg/
116Disallow: /sites/zurich/
117Disallow: /sites/freeenterprise/
118Disallow: /sites/allclearid/
119Disallow: /sites/accenture/
120Disallow: /sites/mercedesbenz/
121Disallow: /sites/nice/
122Disallow: /sites/riverbed/
123Disallow: /sites/pimco/
124Disallow: /sites/netjets/
125Disallow: /sites/vmware/
126Disallow: /sites/visa/
127Disallow: /sites/statefarm/
128Disallow: /sites/philips/
129Disallow: /sites/adp/
130Disallow: /sites/zenithwatches/
131Disallow: /sites/citrix/
132Disallow: /sites/gwcic/
133Disallow: /sites/cargill/
134Disallow: /sites/bankofsingapore/
135Disallow: /sites/dbsbank/
136Disallow: /sites/gradsoflife/
137Disallow: /sites/internationaliotcouncil/
138Disallow: /pega/customers-for-life
139Disallow: /celebritycruises/guide-to-the-globe
140Disallow: /sites/disneyinstitute/
141Disallow: /sites/concertocloud/
142Disallow: /sites/capitalonefutureedge/
143Disallow: /sites/mahindracomviva/
144Disallow: /sites/workday/
145Disallow: /sites/colorado/
146Disallow: /sites/charlesschwab/
147Disallow: /sites/chicagobooth/
148Disallow: /sites/morganstanley/
149Disallow: /sites/opentext/
150Disallow: /sites/icims/
151Disallow: /sites/thriventmutualfunds/
152Disallow: /sites/cbre/
153Disallow: /sites/universityofphoenix/#57934e047260
154Disallow: /sites/ppg/
155Disallow: /sites/americanfunds/
156Disallow: /sites/lilly/
157Disallow: /sites/norton/
158Disallow: /sites/qualityassurance/
159Disallow: /sites/qlik/
160Disallow: /whats-next/planes-trains-and-analytics
161Disallow: /top-50-wealth-managers
162Disallow: /toyota/fueled-by-skepticism
163Disallow: /toyota/coming-soon
164Disallow: /dell/cloud-future-of-medicine
165Disallow: /cit/renewable-energy
166Disallow: /retail
167Disallow: /mobile/cities-transformed
168Disallow: /cit/small-business-mergers-2
169Disallow: /cit/small-business-mergers
170Disallow: /sites/kpmg/
171Disallow: /dell/cities-transformed
172Disallow: /sites/neustar
173Disallow: /sites/ubs
174Disallow: /sites/northerntrust
175Disallow: /sites/infiniti
176Disallow: /sites/honeywell
177Disallow: /sites/charlesschwabadvisor
178Disallow: /sites/frisco
179Disallow: /sites/colehaan
180Disallow: /sites/gapincthiswayahead
181Disallow: /sites/discoverpersonalloans
182Disallow: /sites/delltechnologies
183Disallow: /colehaan/daring-to-step-forward
184Disallow: /johnhancock/
185Disallow: /johnhancock/in-a-lifetime-healthcare
186Disallow: /johnhancock/in-a-lifetime-work
187Disallow: /oppenheimerfunds/the-upside
188Disallow: /sites/hpeenterprisenxt
189Disallow: /sites/rackspace
190Disallow: /sites/themacallan
191Disallow: /themacallan/worlds-together
192Disallow: /sites/inteliotpartnership
193Disallow: /oppenheimerfunds/the-upside
194Disallow: /partners/
195Disallow: /johnhancock/in-a-lifetime-sustainability
196Disallow: /sites/domperignon
197Disallow: /sites/shinola
198Disallow: /sites/hpinc
199Disallow: /sites/hp
200Disallow: /sites/alfresco
201Disallow: /cbre/built-for-performance
202Disallow: /sites/mmedric
203Disallow: /sites/indiana
204Disallow: /sites/siemenshealthineers
205Disallow: /sites/rackspace
206Disallow: /rackspace/elevating-performance
207Disallow: /sites/suntrust
208Disallow: /sites/unicefusa
209Disallow: /sites/mitsubishiheavyindustries
210Disallow: /best-countries-for-business/list
211Disallow: /sites/paloaltonetworks
212Disallow: /sites/f5
213Disallow: /sites/samsungtv
214Disallow: /sites/qa
215Disallow: /sites/paycom
216Disallow: /sites/testbrandvoice
217Disallow: /business/blahblah
218Disallow: /sites/spdr
219Disallow: /sites/lincoln
220Disallow: /sites/abb
221Disallow: /sites/eycybersecurity
222Disallow: /sites/civicnation
223Disallow: /sites/bcg
224Disallow: /sites/vanguard
225Disallow: /sites/akamai
226Disallow: /sites/kyocera
227Disallow: /sites/socialimpact
228Disallow: /sites/emirates
229Disallow: /sites/hpincemea
230Disallow: /sites/ibmperformancecomputing
231Disallow: /sites/pfizer
232Disallow: /sites/ibmpowersystems
233Disallow: /sites/vanguardcharitable
234Disallow: /sites/suntrustprivatewealth
235Disallow: /sites/capitalone
236Disallow: /sites/juniper
237Disallow: /sites/wellsfargo
238Disallow: /sites/bnymellon
239Disallow: /sites/braintree
240Disallow: /sites/bnymellonwealthmanagement
241Disallow: /sites/epson
242Disallow: /sites/thomsonreuters
243Disallow: /sites/paysafe
244Disallow: /sites/josabank
245Disallow: /sites/hennessyparadisimperial
246Disallow: /sites/cartier
247Disallow: /sites/anabluewing
248Disallow: /sites/eidoo
249Disallow: /sites/cmstesting
250Disallow: /sites/adobepartnership
251Disallow: /sites/cornelljohnson
252Disallow: /sites/quantcast
253Disallow: /sites/japan
254Disallow: /sites/impactpartners
255Disallow: /sites/nvidia
256Disallow: /sites/personalcapital
257Disallow: /sites/microsoft365
258Disallow: /sites/providencedigitalinnovation
259Disallow: /sites/whittiertrust
260Disallow: /sites/kinetica
261Disallow: /sites/paulevans
262Disallow: /sites/ritz-carltonleadershipcenter
263Disallow: /sites/icoengine
264Disallow: /sites/tesaro
265Disallow: /sites/theglenlivet
266Disallow: /feature/sf-test
267Disallow: /sites/courtyardmarriott
268Disallow: /sites/nuveen
269Disallow: /sites/intelai
270Disallow: /sites/secret
271Disallow: /sites/hec
272Disallow: /retail/old
273Disallow: /sites/alliantgroup
274Disallow: /sites/intuitivesurgical
275Disallow: /sites/sunovion
276Disallow: /sites/ofx
277Disallow: /sites/firstrepublicbank
278Disallow: /sites/1850byfolgers
279Disallow: /sites/audi
280Disallow: /sites/nuveencef
281Disallow: /sites/1850brandcoffee
282Disallow: /sites/cisco
283Disallow: /sites/blackstarcapitalltd
284Disallow: /sites/usbankaltitudereserve
285Disallow: /sites/dassaultsystemes
286Disallow: /sites/urogen
287Disallow: /sites/smartsheet
288Disallow: /sites/iwc
289Disallow: /sites/datingcom
290Disallow: /sites/intuitquickbooks
291Disallow: /sites/oticon
292Disallow: /sites/servicenow
293Disallow: /sites/deloitteriskandfinancialadvisory
294Disallow: /sites/dbsprivatebank
295Disallow: /sites/americanexpress
296Disallow: /sites/tanotofoundation
297Disallow: /sites/citizensaccess
298Disallow: /sites/salesforceessentials
299Disallow: /sites/drexeluniversity
300Disallow: /sites/microsoftinnovation
301Disallow: /sites/mypos
302Disallow: /sites/harvardpilgrimhealthcare
303Disallow: /sites/appdynamics
304Disallow: /sites/ministryoftourismindia
305Disallow: /sites/orlando
306Disallow: /sites/extrahop
307Disallow: /sites/ascm
308Disallow: /sites/crowe
309Disallow: /sites/peerfit
310Disallow: /sites/theleukemialymphomasociety
311Disallow: /sites/hewlettpackardenterprise
312Disallow: /sites/montblanc
313Disallow: /sites/meero
314Disallow: /sites/braze
315Disallow: /sites/publicissapient
316Disallow: /sites/flexport
317
318User-agent: Amazonbot
319Disallow: /
320
321User-agent: anthropic-ai
322Disallow: /
323
324User-agent: Applebot-Extended
325Disallow: /
326
327User-agent: Bytespider
328Disallow: /
329
330User-agent: CCBot
331User-agent: CCbot
332Disallow: /
333
334User-agent: ClaudeBot
335Disallow: /
336
337User-agent: cohere-ai
338User-agent: Cohere-AI
339Disallow: /
340
341User-agent: Diffbot
342Disallow: /
343
344User-agent: FacebookBot
345Disallow: /
346
347User-agent: FriendlyCrawler
348Disallow: /
349
350User-agent: GPTBot
351Disallow: /
352
353User-agent: ImagesiftBot
354Disallow: /
355
356User-agent: meta-externalagent
357User-agent: Meta-ExternalAgent
358Disallow: /
359
360User-Agent: OAI-SearchBot
361Allow: /
362
363User-agent: omgili
364Disallow: /
365
366User-agent: omgilibot
367Disallow: /
368
369User-Agent: PerplexityBot
370Disallow: /
371
372User-agent: AhrefsBot
373User-agent: AhrefsSiteAudit
374Crawl-delay: 2
375
376User-Agent: Claude-User
377User-Agent: Claude-SearchBot
378Allow: /
379
380User-agent: AmazonAdBot
381Allow: /following/
382Allow: /search/
383Allow: /follow/
384Allow: /author/following/
385Allow: /blog/following/
386Allow: /header/channels/
387Allow: /find-more/
388Allow: /preview/
389Allow: /typeahead/
390Allow: /pepe/
391Allow: /media-manager/
392Allow: /coupons/visit/*
393Allow: /advisor/*zx1fa*
394Allow: /wui/
395
396Sitemap: https://www.forbes.com/news_sitemap.xml
397Sitemap: https://www.forbes.com/video_sitemap.xml
398Sitemap: https://www.forbes.com/companies_sitemap.xml
399Sitemap: https://www.forbes.com/health/sitemap.xml
400Sitemap: https://www.forbes.com/connect/events-sitemap.xml
401Sitemap: https://www.forbes.com/advisor/sitemap.xml
402Sitemap: https://www.forbes.com/home-improvement/sitemap.xml
403Sitemap: https://www.forbes.com/digital-assets/sitemap.xml