NeuralCrawl

IBM / robots.txt snapshot

← back to ibm.com · fetched 2026-09-24T23:05:48Z (1mo ago) · HTTP 200 · 4616 bytes · sha256 8e29d2cf6f21b8d8 · raw

final URL: https://www.ibm.com/robots.txt

1User-agent: *
2Disallow: /account/registration
3Disallow: /account/mypro
4Disallow: /account/myint
5Disallow: /Admin
6Disallow: /cgi-
7Disallow: /contact/employees/servlets
8Disallow: /data/
9Disallow: /db2s
10Disallow: /developerworks/*-pdf.pdf$
11Disallow: /developerworks/forums/servlet
12Disallow: /developerworks/forums/abuse
13Disallow: /developerworks/forums/post
14Disallow: /docs/api
15Disallow: /fcgi-
16Disallow: /fscripts
17# Disallow: /homepage
18Disallow: /image
19Disallow: /mashupmaker
20Disallow: /mysupport/s/forumshome
21Disallow: /mysupport/s/forumsproduct
22Disallow: /mysupport/s/forumsquestion
23Disallow: /partnerworld/portal
24Disallow: /portal/home
25Disallow: /products/finder
26Disallow: /profiles
27Disallow: /scripts
28Disallow: /Scripts
29Disallow: /search/?
30Disallow: /Search
31Disallow: /smartmarket/*/productListInitial.action?
32Disallow: /standards
33Disallow: /support/knowledgecenter/api
34Disallow: /support/knowledgecenter/v1
35Disallow: /survey
36Disallow: /testing
37Disallow: /tmp
38Disallow: /vrm/bin/sendThisPage
39Disallow: /webmaster
40Disallow: /zx
41Disallow: /zz
42Disallow: /*.ssi$
43Disallow: /account/myibm/InterestsEdit.do
44Disallow: /wcs
45Disallow: /wcsstore
46Disallow: /webapp
47Disallow: /web/portal/software/websphere
48Disallow: /common/austin-summit
49Disallow: /link
50Disallow: /links
51Disallow: /web/portal/commerce
52Disallow: /industries/clients
53Disallow: /standards
54# Disallow: /internal
55Disallow: /test
56Disallow: /test-kick-tires
57Disallow: /test-v1-page-template
58Disallow: /product-detail
59Disallow: /watson/assets/pdfs/reg
60Disallow: /zprod-ease
61Disallow: /zProd-ease
62Disallow: /zshowcase
63Disallow: /widgets-admin
64Disallow: /w3
65Disallow: /mintlify-assets
66Disallow: /industries/federal/fedcat/ibm-automation-services-for-government
67# Disallow: /multimedia
68
69sitemap: https://www.ibm.com/content/sitemap/href-sitemap-index.xml
70sitemap: https://www.ibm.com/content/dam/adobe-cms/IBM_Adobe_Sitemap.xml
71sitemap: https://www.ibm.com/mysupport/s/sitemap.xml
72sitemap: https://www.ibm.com/us-en.sitemap.news-sitemap.xml
73sitemap: https://www.ibm.com/docs/en/sitemap.xml
74
75User-agent: Fast corporate crawler
76Disallow: //
77Disallow: /account/registration
78Disallow: /account/mypro
79Disallow: /account/myint
80Disallow: /Admin
81Disallow: /cgi-
82Disallow: /common
83Disallow: /contact/employees/servlets
84Disallow: /data/
85Disallow: /db2s
86Disallow: /developerworks
87Disallow: /fcgi-
88Disallow: /fscripts
89Disallow: /homepage
90Disallow: /i/
91Disallow: /image
92Disallow: /investor
93Disallow: /link
94Disallow: /mashupmaker
95Disallow: /portal/home
96Disallow: /products/finder
97Disallow: /profiles
98Disallow: /SametimeWebApp
99Disallow: /scripts
100Disallow: /Scripts
101Disallow: /search/?
102Disallow: /Search
103Disallow: /survey
104Disallow: /tmp
105Disallow: /vrm/bin/sendThisPage
106Disallow: /webmaster
107Disallow: /zx
108Disallow: /zz
109
110User-agent: 008
111Disallow: /smartmarket
112
113# Added for IN4173782 on 7Aug2013
114User-agent: magpie-crawler
115Disallow: /connections/
116
117# Added for IN4177562 on 8Aug2013
118User-agent: magpie-crawler/1.1
119Disallow: /connections/
120
121# Added for IN4177562 on 8Aug2013
122User-agent: MagpieRSS
123Disallow: /connections/
124
125User-agent: psnsearch
126Disallow: /developerworks/wikis/login.action
127
128# Added to block site mirroring
129User-agent: HTTrack
130Disallow: /
131
132
133User-agent: Acrolinx-bot
134# Disallow: //
135Disallow: /account/registration
136Disallow: /account/mypro
137Disallow: /account/myint
138Disallow: /Admin
139Disallow: /cgi-
140Disallow: /contact/employees/servlets
141Disallow: /data/
142Disallow: /db2s
143Disallow: /developerworks/*-pdf.pdf$
144Disallow: /developerworks/forums/servlet
145Disallow: /developerworks/forums/abuse
146Disallow: /developerworks/forums/post
147Disallow: /docs/api
148Disallow: /fcgi-
149Disallow: /fscripts
150# Disallow: /homepage
151Disallow: /image
152Disallow: /mashupmaker
153Disallow: /mysupport/s/forumshome
154Disallow: /mysupport/s/forumsproduct
155Disallow: /mysupport/s/forumsquestion
156Disallow: /partnerworld/portal
157Disallow: /portal/home
158Disallow: /products/finder
159Disallow: /profiles
160Disallow: /scripts
161Disallow: /Scripts
162Disallow: /search/?
163Disallow: /Search
164Disallow: /smartmarket/*/productListInitial.action?
165Disallow: /standards
166Disallow: /support/knowledgecenter/api
167Disallow: /support/knowledgecenter/v1
168Disallow: /survey
169Disallow: /testing
170Disallow: /tmp
171Disallow: /vrm/bin/sendThisPage
172Disallow: /webmaster
173Disallow: /zx
174Disallow: /zz
175Disallow: /*.ssi$
176Disallow: /account/myibm/InterestsEdit.do
177Disallow: /wcs
178Disallow: /wcsstore
179Disallow: /webapp
180Disallow: /web/portal/software/websphere
181Disallow: /common/austin-summit
182Disallow: /link
183Disallow: /links
184Disallow: /web/portal/commerce
185Disallow: /industries/clients
186Disallow: /standards
187
188User-agent: amazon-Quick-on-behalf-of
189Disallow: /mysupport