NeuralCrawl

DBS Group / robots.txt snapshot

← back to dbs.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 5648 bytes · sha256 462ba58f3b162e5f · raw

final URL: https://www.dbs.com/robots.txt

1User-agent: Googlebot
2Allow: /iwov-resources/
3
4User-agent:*
5Allow: /
6Disallow: /campaigns/
7Disallow: /g22/
8Disallow: /g23/
9Disallow: /g24/
10Disallow: /g25/
11Disallow: /g26/
12Disallow: /g27/
13Disallow: /microsite/ecards/
14Disallow: /NewsPrinter.page
15
16
17Sitemap: https://www.dbs.com/sitemap.xml
18Sitemap: https://www.dbs.com/gsmc_sitemap.xml
19Sitemap: https://www.dbs.com/private-banking/sitemap-privatebanking.xml
20
21
22User-agent: OmniExplorer-Bot
23Disallow: /
24
25User-agent: FreeFind
26Disallow: /
27
28User-agent: BecomeBot
29Disallow: /
30
31User-agent: Nutch
32Disallow: /
33
34User-agent: Jetbot-1
35Disallow: /
36
37User-agent: Jetbot
38Disallow: /
39
40User-agent: WebVac
41Disallow: /
42
43User-agent: Stanford
44Disallow: /
45
46User-agent: naver
47Disallow: /
48
49User-agent: dumbot
50Disallow: /
51
52User-agent: Hatena-Antenna
53Disallow: /
54
55User-agent: grub-client
56Disallow: /
57
58User-agent: grub
59Disallow: /
60
61User-agent: WebZip
62Disallow: /
63
64User-agent: larbin
65Disallow: /
66
67User-agent: b2w-01
68Disallow: /
69
70User-agent: Copernic
71Disallow: /
72
73User-agent: psbot
74Disallow: /
75
76User-agent: Python-urllib
77Disallow: /
78
79User-agent: NetMechanic
80Disallow: /
81
82User-agent: URL_Spider_Pro
83Disallow: /
84
85User-agent: CherryPicker
86Disallow: /
87
88User-agent: EmailCollector
89Disallow: /
90
91User-agent: EmailSiphon
92Disallow: /
93
94User-agent: WebBandit
95Disallow: /
96
97User-agent: EmailWolf
98Disallow: /
99
100User-agent: ExtractorPro
101Disallow: /
102
103User-agent: CopyRightCheck
104Disallow: /
105
106User-agent: Crescent
107Disallow: /
108
109User-agent: SiteSnagger
110Disallow: /
111
112User-agent: ProWebWalker
113Disallow: /
114
115User-agent: CheeseBot
116Disallow: /
117
118User-agent: LNSpiderguy
119Disallow: /
120
121User-agent: Alexibot
122Disallow: /
123
124User-agent: Teleport
125Disallow: /
126
127User-agent: TeleportPro
128Disallow: /
129
130User-agent: Stanford-Comp-Sci
131Disallow: /
132
133User-agent: MIIxpc
134Disallow: /
135
136User-agent: Telesoft
137Disallow: /
138
139User-agent: Website-Quester
140Disallow: /
141
142User-agent: moget-2-1
143Disallow: /
144
145User-agent: WebZip-4-0
146Disallow: /
147
148User-agent: WebStripper
149Disallow: /
150
151User-agent: WebSauger
152Disallow: /
153
154User-agent: WebCopier
155Disallow: /
156
157User-agent: NetAnts
158Disallow: /
159
160User-agent: Mister-PiX
161Disallow: /
162
163User-agent: WebAuto
164Disallow: /
165
166User-agent: TheNomad
167Disallow: /
168
169User-agent: WWW-Collector-E
170Disallow: /
171
172User-agent: RMA
173Disallow: /
174
175User-agent: libWeb-clsHTTP
176Disallow: /
177
178User-agent: asterias
179Disallow: /
180
181User-agent: httplib
182Disallow: /
183
184User-agent: turingos
185Disallow: /
186
187User-agent: spanner
188Disallow: /
189
190User-agent: InfoNaviRobot
191Disallow: /
192
193User-agent: Harvest-1.5
194Disallow: /
195
196User-agent: Bullseye-1.0
197Disallow: /
198
199User-agent: Crescent-Internet-ToolPak-HTTP-OLE-Control-v-1-0
200Disallow: /
201
202User-agent: CherryPickerSE-1-0
203Disallow: /
204
205User-agent: CherryPickerElite-1-0
206Disallow: /
207
208User-agent: WebBandit-3-50
209Disallow: /
210
211User-agent: NICErsPRO
212Disallow: /
213
214User-agent: Microsoft-URL-Control-5-01-4511
215Disallow: /
216
217User-agent: DittoSpyder
218Disallow: /
219
220User-agent: Foobot
221Disallow: /
222
223User-agent: SpankBot
224Disallow: /
225
226User-agent: BotALot
227Disallow: /
228
229User-agent: lwp-trivial-1-34
230Disallow: /
231
232User-agent: lwp-trivial
233Disallow: /
234
235User-agent: BunnySlippers
236Disallow: /
237
238User-agent: Microsoft-URL-Control-6-00-8169
239Disallow: /
240
241User-agent: URLy-Warning
242Disallow: /
243
244User-agent: Wget-1-6
245Disallow: /
246
247User-agent: Wget-1-5-3
248Disallow: /
249
250User-agent: Wget
251Disallow: /
252
253User-agent: LinkWalker
254Disallow: /
255
256User-agent: cosmos
257Disallow: /
258
259User-agent: moget
260Disallow: /
261
262User-agent: hloader
263Disallow: /
264
265User-agent: humanlinks
266Disallow: /
267
268User-agent: LinkextractorPro
269Disallow: /
270
271User-agent: Offline-Explorer
272Disallow: /
273
274User-agent: Mata-Hari
275Disallow: /
276
277User-agent: LexiBot
278Disallow: /
279
280User-agent: Web-Image-Collector
281Disallow: /
282
283User-agent: The-Intraformant
284Disallow: /
285
286User-agent: True_Robot-1-0
287Disallow: /
288
289User-agent: True_Robot
290Disallow: /
291
292User-agent: BlowFish-1-0
293Disallow: /
294
295User-agent: JennyBot
296Disallow: /
297
298User-agent: MIIxpc-4-2
299Disallow: /
300
301User-agent: BuiltBotTough
302Disallow: /
303
304User-agent: ProPowerBot-2-14
305Disallow: /
306
307User-agent: BackDoorBot-1-0
308Disallow: /
309
310User-agent: toCrawl-UrlDispatcher
311Disallow: /
312
313User-agent: WebEnhancer
314Disallow: /
315
316User-agent: suzuran
317Disallow: /
318
319User-agent: VCI-WebViewer-VCI-WebViewer-Win32
320Disallow: /
321
322User-agent: VCI
323Disallow: /
324
325User-agent: Szukacz-1-4
326Disallow: /
327
328User-agent: QueryN-Metasearch
329Disallow: /
330
331User-agent: Openfind-data-gathere
332Disallow: /
333
334User-agent: Openfind
335Disallow: /
336
337User-agent: Xenus-Link-Sleuth-1-1c
338Disallow: /
339
340User-agent: Xenus
341Disallow: /
342
343User-agent: Zeus
344Disallow: /
345
346User-agent: RepoMonkey-Bait-Tackle-v1-01
347Disallow: /
348
349User-agent: RepoMonkey
350Disallow: /
351
352User-agent: Microsoft-URL-Control
353Disallow: /
354
355User-agent: Openbot
356Disallow: /
357
358User-agent: URL-Control
359Disallow: /
360
361User-agent: Zeus-Link-Scout
362Disallow: /
363
364User-agent: Zeus-32297-Webster-Pro-V2-9-Win32
365Disallow: /
366
367User-agent: Webster-Pro
368Disallow: /
369
370User-agent: EroCrawler
371Disallow: /
372
373User-agent: LinkScan-8-1a-Unix
374Disallow: /
375
376User-agent: Keyword-Density-0-9
377Disallow: /
378
379User-agent: Kenjin-Spider
380Disallow: /
381
382User-agent: Iron33-1-0-2
383Disallow: /
384
385User-agent: Bookmark-search-tool
386Disallow: /
387
388User-agent: GetRight-4-2
389Disallow: /
390
391User-agent: FairAd-Client
392Disallow: /
393
394User-agent: Gaisbot
395Disallow: /
396
397User-agent: Aqua_Products
398Disallow: /
399
400User-agent: Radiation-Retriever-1-1
401Disallow: /
402
403User-agent: Flaming-AttackBot
404Disallow: /
405
406User-agent: Oracle-Ultra-Search
407Disallow: /
408
409User-agent: MSIECrawler
410Disallow: /
411
412User-agent: PerMan
413Disallow: /
414
415User-agent: searchpreview
416Disallow: /
417
418User-agent: sootle
419Disallow: /
420
421User-agent: es
422Disallow: /
423
424User-agent: Enterprise-Search-1-0
425Disallow: /
426
427User-agent: Enterprise_Search
428Disallow: /
429
430User-agent: Livemore
431Disallow: /livemore/vl/
432
433User-agent: bitlybot
434Disallow: /