NeuralCrawl

IBM / robots.txt snapshot

← back to ibm.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 4540 bytes · sha256 764391437703b48b · raw

final URL: https://www.ibm.com/robots.txt

1User-agent: *
2Disallow: /account/registration
3Disallow: /account/mypro
4Disallow: /account/myint
5Disallow: /Admin
6Disallow: /cgi-
7Disallow: /contact/employees/servlets
8Disallow: /data/
9Disallow: /db2s
10Disallow: /developerworks/*-pdf.pdf$
11Disallow: /developerworks/forums/servlet
12Disallow: /developerworks/forums/abuse
13Disallow: /developerworks/forums/post
14Disallow: /docs/api
15Disallow: /fcgi-
16Disallow: /fscripts
17# Disallow: /homepage
18Disallow: /image
19Disallow: /mashupmaker
20Disallow: /mysupport/s/forumshome
21Disallow: /mysupport/s/forumsproduct
22Disallow: /mysupport/s/forumsquestion
23Disallow: /partnerworld/portal
24Disallow: /portal/home
25Disallow: /products/finder
26Disallow: /profiles
27Disallow: /scripts
28Disallow: /Scripts
29Disallow: /search/?
30Disallow: /Search
31Disallow: /smartmarket/*/productListInitial.action?
32Disallow: /standards
33Disallow: /support/knowledgecenter/api
34Disallow: /support/knowledgecenter/v1
35Disallow: /survey
36Disallow: /testing
37Disallow: /tmp
38Disallow: /vrm/bin/sendThisPage
39Disallow: /webmaster
40Disallow: /zx
41Disallow: /zz
42Disallow: /*.ssi$
43Disallow: /account/myibm/InterestsEdit.do
44Disallow: /wcs
45Disallow: /wcsstore
46Disallow: /webapp
47Disallow: /web/portal/software/websphere
48Disallow: /common/austin-summit
49Disallow: /link
50Disallow: /links
51Disallow: /web/portal/commerce
52Disallow: /industries/clients
53Disallow: /standards
54# Disallow: /internal
55Disallow: /test
56Disallow: /test-kick-tires
57Disallow: /test-v1-page-template
58Disallow: /product-detail
59Disallow: /watson/assets/pdfs/reg
60Disallow: /zprod-ease
61Disallow: /zProd-ease
62Disallow: /zshowcase
63Disallow: /widgets-admin
64Disallow: /w3
65Disallow: /mintlify-assets
66# Disallow: /multimedia
67
68sitemap: https://www.ibm.com/content/sitemap/href-sitemap-index.xml
69sitemap: https://www.ibm.com/content/dam/adobe-cms/IBM_Adobe_Sitemap.xml
70sitemap: https://www.ibm.com/mysupport/s/sitemap.xml
71sitemap: https://www.ibm.com/us-en.sitemap.news-sitemap.xml
72sitemap: https://www.ibm.com/docs/en/sitemap.xml
73
74User-agent: Fast corporate crawler
75Disallow: //
76Disallow: /account/registration
77Disallow: /account/mypro
78Disallow: /account/myint
79Disallow: /Admin
80Disallow: /cgi-
81Disallow: /common
82Disallow: /contact/employees/servlets
83Disallow: /data/
84Disallow: /db2s
85Disallow: /developerworks
86Disallow: /fcgi-
87Disallow: /fscripts
88Disallow: /homepage
89Disallow: /i/
90Disallow: /image
91Disallow: /investor
92Disallow: /link
93Disallow: /mashupmaker
94Disallow: /portal/home
95Disallow: /products/finder
96Disallow: /profiles
97Disallow: /SametimeWebApp
98Disallow: /scripts
99Disallow: /Scripts
100Disallow: /search/?
101Disallow: /Search
102Disallow: /survey
103Disallow: /tmp
104Disallow: /vrm/bin/sendThisPage
105Disallow: /webmaster
106Disallow: /zx
107Disallow: /zz
108
109User-agent: 008
110Disallow: /smartmarket
111
112# Added for IN4173782 on 7Aug2013
113User-agent: magpie-crawler
114Disallow: /connections/
115
116# Added for IN4177562 on 8Aug2013
117User-agent: magpie-crawler/1.1
118Disallow: /connections/
119
120# Added for IN4177562 on 8Aug2013
121User-agent: MagpieRSS
122Disallow: /connections/
123
124User-agent: psnsearch
125Disallow: /developerworks/wikis/login.action
126
127# Added to block site mirroring
128User-agent: HTTrack
129Disallow: /
130
131
132User-agent: Acrolinx-bot
133# Disallow: //
134Disallow: /account/registration
135Disallow: /account/mypro
136Disallow: /account/myint
137Disallow: /Admin
138Disallow: /cgi-
139Disallow: /contact/employees/servlets
140Disallow: /data/
141Disallow: /db2s
142Disallow: /developerworks/*-pdf.pdf$
143Disallow: /developerworks/forums/servlet
144Disallow: /developerworks/forums/abuse
145Disallow: /developerworks/forums/post
146Disallow: /docs/api
147Disallow: /fcgi-
148Disallow: /fscripts
149# Disallow: /homepage
150Disallow: /image
151Disallow: /mashupmaker
152Disallow: /mysupport/s/forumshome
153Disallow: /mysupport/s/forumsproduct
154Disallow: /mysupport/s/forumsquestion
155Disallow: /partnerworld/portal
156Disallow: /portal/home
157Disallow: /products/finder
158Disallow: /profiles
159Disallow: /scripts
160Disallow: /Scripts
161Disallow: /search/?
162Disallow: /Search
163Disallow: /smartmarket/*/productListInitial.action?
164Disallow: /standards
165Disallow: /support/knowledgecenter/api
166Disallow: /support/knowledgecenter/v1
167Disallow: /survey
168Disallow: /testing
169Disallow: /tmp
170Disallow: /vrm/bin/sendThisPage
171Disallow: /webmaster
172Disallow: /zx
173Disallow: /zz
174Disallow: /*.ssi$
175Disallow: /account/myibm/InterestsEdit.do
176Disallow: /wcs
177Disallow: /wcsstore
178Disallow: /webapp
179Disallow: /web/portal/software/websphere
180Disallow: /common/austin-summit
181Disallow: /link
182Disallow: /links
183Disallow: /web/portal/commerce
184Disallow: /industries/clients
185Disallow: /standards
186
187User-agent: amazon-Quick-on-behalf-of
188Disallow: /mysupport