NeuralCrawl

UnitedHealth Group / robots.txt snapshot

← back to unitedhealthgroup.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 6860 bytes · sha256 d2b2860c11903b43 · raw

final URL: https://unitedhealthgroup.com/robots.txt

1#
2# robots.txt file
3#
4#
5User-agent: Twitterbot
6Disallow: /
7
8Allow: /content/dam/uhg/Images/
9Allow: /content/dam/uhg/icons/
10
11# XML Sitemap Location
12Sitemap: https://www.unitedhealthgroup.com/content/uhg/en.sitemap.xml
13
14User-agent: *
15
16Disallow: /~/search.html
17Disallow: /content/uhg/en/ns/
18Disallow: /content/dam/sustainability
19Disallow: /content/dam/sustainability-report
20Disallow: /content/dam/sparq
21Disallow: /content/dam/templates
22Disallow: /content/dam/catalogs
23Disallow: /content/dam/projects
24Disallow: /content/dam/UHG/protectwell
25Disallow: /content/dam/UHG/PDF/uhgwebcomponents
26Disallow: /content/sustainability
27Disallow: /content/darksite
28Disallow: /content/sparq
29Disallow: /content/huge
30Disallow: /content/dam/UHG/PDF/ns/
31Disallow: /content/dam/UHG/PDF/PurchaseOrder
32Disallow: /content/dam/UHG/PDF/site
33Disallow: /content/dam/ma
34Disallow: /admin/
35Disallow: /crx/
36Disallow: /system/
37Disallow: /apps/
38Disallow: /bin/
39Disallow: /libs/
40Disallow: /home/
41Disallow: /tmp/
42Disallow: /var/
43Disallow: /etc/
44
45# Unsafe robots to disallow
46
47User-agent: psbot
48Disallow: /
49
50User-agent: Aqua_Products
51Disallow: /
52
53User-agent: asterias
54Disallow: /
55
56User-agent: b2w/0.1
57Disallow: /
58
59User-agent: BackDoorBot/1.0
60Disallow: /
61
62User-agent: Black Hole
63Disallow: /
64
65User-agent: BlowFish/1.0
66Disallow: /
67
68User-agent: Bookmark search tool
69Disallow: /
70
71User-agent: BotALot
72Disallow: /
73
74User-agent: BuiltBotTough
75Disallow: /
76
77User-agent: Bullseye/1.0
78Disallow: /
79
80User-agent: BunnySlippers
81Disallow: /
82
83User-agent: Cegbfeieh
84Disallow: /
85
86User-agent: CheeseBot
87Disallow: /
88
89User-agent: CherryPicker
90Disallow: /
91
92User-agent: CherryPicker /1.0
93Disallow: /
94
95User-agent: CherryPickerElite/1.0
96Disallow: /
97
98User-agent: CherryPickerSE/1.0
99Disallow: /
100
101User-agent: CopyRightCheck
102Disallow: /
103
104User-agent: cosmos
105Disallow: /
106
107User-agent: Crescent
108Disallow: /
109
110User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
111Disallow: /
112
113User-agent: DittoSpyder
114Disallow: /
115
116User-agent: EmailCollector
117Disallow: /
118
119User-agent: EmailSiphon
120Disallow: /
121
122User-agent: EmailWolf
123Disallow: /
124
125User-agent: EroCrawler
126Disallow: /
127
128User-agent: ExtractorPro
129Disallow: /
130
131User-agent: FairAd Client
132Disallow: /
133
134User-agent: Flaming AttackBot
135Disallow: /
136
137User-agent: Foobot
138Disallow: /
139
140User-agent: Gaisbot
141Disallow: /
142
143User-agent: GetRight/4.2
144Disallow: /
145
146User-agent: gomezagent
147Disallow: /
148User-agent: GomezAgent
149Disallow: /
150User-Agent: gomezagent+1.0
151Disallow: /
152User-Agent: GomezAgent+1.0
153Disallow: /
154User-Agent: gomezagent+2.0
155Disallow: /
156User-Agent: GomezAgent+2.0
157Disallow: /
158User-Agent: gomezagent+3.0
159Disallow: /
160User-Agent: GomezAgent+3.0
161Disallow: /
162User-agent: gomezagent 1.0
163Disallow: /
164User-agent: GomezAgent 1.0
165Disallow: /
166User-agent: gomezagent 2.0
167Disallow: /
168User-agent: GomezAgent 2.0
169Disallow: /
170User-agent: gomezagent 3.0
171Disallow: /
172User-Agent: GomezAgent 3.0
173Disallow: /
174
175User-agent: grub
176Disallow: /
177
178User-agent: grub-client
179Disallow: /
180
181User-agent: Harvest/1.5
182Disallow: /
183
184User-agent: hloader
185Disallow: /
186
187User-agent: httplib
188Disallow: /
189
190User-agent: humanlinks
191Disallow: /
192
193User-agent: ia_archiver
194Disallow: /
195
196User-agent: ia_archiver/1.6
197Disallow: /
198
199User-agent: InfoNaviRobot
200Disallow: /
201
202User-agent: Iron33/1.0.2
203Disallow: /
204
205User-agent: JennyBot
206Disallow: /
207
208User-agent: Kenjin Spider
209Disallow: /
210
211User-agent: Keyword Density/0.9
212Disallow: /
213
214User-agent: larbin
215Disallow: /
216
217User-agent: LexiBot
218Disallow: /
219
220User-agent: libWeb/clsHTTP
221Disallow: /
222
223User-agent: LinkextractorPro
224Disallow: /
225
226User-agent: LinkScan/8.1a Unix
227Disallow: /
228
229User-agent: LinkWalker
230Disallow: /
231
232User-agent: LNSpiderguy
233Disallow: /
234
235User-agent: lwp-trivial
236Disallow: /
237
238User-agent: lwp-trivial/1.34
239Disallow: /
240
241User-agent: Mata Hari
242Disallow: /
243
244User-agent: Microsoft URL Control
245Disallow: /
246
247User-agent: Microsoft URL Control - 5.01.4511
248Disallow: /
249
250User-agent: Microsoft URL Control - 6.00.8169
251Disallow: /
252
253User-agent: MIIxpc
254Disallow: /
255
256User-agent: MIIxpc/4.2
257Disallow: /
258
259User-agent: Mister PiX
260Disallow: /
261
262User-agent: moget
263Disallow: /
264
265User-agent: moget/2.1
266Disallow: /
267
268User-agent: mozilla/4
269Disallow: /
270
271User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
272Disallow: /
273
274User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
275Disallow: /
276
277User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
278Disallow: /
279
280User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
281Disallow: /
282
283User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)
284Disallow: /
285
286User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
287Disallow: /
288
289User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
290Disallow: /
291
292User-agent: mozilla/5
293Disallow: /
294
295User-agent: MSIECrawler
296Disallow: /
297
298User-agent: NetAnts
299Disallow: /
300
301User-agent: NetMechanic
302Disallow: /
303
304User-agent: NICErsPRO
305Disallow: /
306
307User-agent: Offline Explorer
308Disallow: /
309
310User-agent: Openbot
311Disallow: /
312
313User-agent: Openfind
314Disallow: /
315
316User-agent: Openfind data gathere
317Disallow: /
318
319User-agent: Oracle Ultra Search
320Disallow: /
321
322User-agent: PerMan
323Disallow: /
324
325User-agent: ProPowerBot/2.14
326Disallow: /
327
328User-agent: ProWebWalker
329Disallow: /
330
331User-agent: Python-urllib
332Disallow: /
333
334User-agent: QueryN Metasearch
335Disallow: /
336
337User-agent: Radiation Retriever 1.1
338Disallow: /
339
340User-agent: RepoMonkey
341Disallow: /
342
343User-agent: RepoMonkey Bait & Tackle/v1.01
344Disallow: /
345
346User-agent: RMA
347Disallow: /
348
349User-agent: searchpreview
350Disallow: /
351
352User-agent: SiteSnagger
353Disallow: /
354
355User-agent: SpankBot
356Disallow: /
357
358User-agent: spanner
359Disallow: /
360
361User-agent: suzuran
362Disallow: /
363
364User-agent: Szukacz/1.4
365Disallow: /
366
367User-agent: Teleport
368Disallow: /
369
370User-agent: TeleportPro
371Disallow: /
372
373User-agent: Telesoft
374Disallow: /
375
376User-agent: The Intraformant
377Disallow: /
378
379User-agent: TheNomad
380Disallow: /
381
382User-agent: TightTwatBot
383Disallow: /
384
385User-agent: Titan
386Disallow: /
387
388User-agent: toCrawl/UrlDispatcher
389Disallow: /
390
391User-agent: True_Robot
392Disallow: /
393
394User-agent: True_Robot/1.0
395Disallow: /
396
397User-agent: turingos
398Disallow: /
399
400User-agent: URL Control
401Disallow: /
402
403User-agent: URL_Spider_Pro
404Disallow: /
405
406User-agent: URLy Warning
407Disallow: /
408
409User-agent: VCI
410Disallow: /
411
412User-agent: VCI WebViewer VCI WebViewer Win32
413Disallow: /
414
415User-agent: Web Image Collector
416Disallow: /
417
418User-agent: WebAuto
419Disallow: /
420
421User-agent: WebBandit
422Disallow: /
423
424User-agent: WebBandit/3.50
425Disallow: /
426
427User-agent: WebCopier
428Disallow: /
429
430User-agent: WebEnhancer
431Disallow: /
432
433User-agent: WebmasterWorldForumBot
434Disallow: /
435
436User-agent: WebSauger
437Disallow: /
438
439User-agent: Website Quester
440Disallow: /
441
442User-agent: Webster Pro
443Disallow: /
444
445User-agent: WebStripper
446Disallow: /
447
448User-agent: Wget
449Disallow: /
450
451User-agent: Wget/1.5.3
452Disallow: /
453
454User-agent: Wget/1.6
455Disallow: /
456
457User-agent: WWW-Collector-E
458Disallow: /
459
460User-agent: Zeus
461Disallow: /
462
463User-agent: Zeus 32297 Webster Pro V2.9 Win32
464Disallow: /
465
466User-agent: Zeus Link Scout
467Disallow: /