NeuralCrawl

Barracuda / robots.txt snapshot

← back to barracuda.com · fetched 2026-06-20T14:27:25Z (7h ago) · HTTP 200 · 5217 bytes · sha256 d646e35db79825a3 · raw

final URL: https://www.barracuda.com/robots.txt

1User-agent: *
2Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
3Crawl-delay: 1
4User-agent: *
5
6# Prevent Crawling for Thank You Pages
7Disallow: */*LP-OnDemand*
8Disallow: */*LP-ThankYou*
9Disallow: */*thankyou*
10Disallow: */*thank-you*
11Disallow: */thankyou/*
12Disallow: */thankyou*
13Disallow: */thankYou*
14
15# Prevent Crawling for Login & Register
16Disallow: */login/*
17Disallow: */register/*
18Disallow: */customers/s/login/*
19Disallow: */auth/login/*
20
21# Prevent Error Crawling for Templates
22Disallow: /error/
23
24# Prevent Crawling for Captcha
25Disallow: */captcha/*
26
27# Prevent Crawling for Admin
28Disallow: */admin/*
29Disallow: */administrator/*
30
31# Prevent Crawling for User
32Disallow: */user/*
33Disallow: */users/*
34
35# Prevent Crawling for CGI
36Disallow: /cgi-bin/
37
38# Prevent Crawling for cache and log
39Disallow: */cache/*
40Disallow: */log/*
41
42
43########## SITEMAP FILES ##########
44Sitemap: https://www.barracuda.com/sitemap.xml
45
46
47
48########## PREVENT BAD BOTS TO CRAWL ##########
49
50# http://www.opensiteexplorer.org/dotbot
51User-agent: *
52Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
53Crawl-delay: 1
54User-agent: DotBot
55User-agent: *
56Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
57Crawl-delay: 1
58User-agent: dotbot
59Disallow: /
60
61# http://help.sogou.com/homepage/search-engine/add-crawl.html
62User-agent: *
63Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
64Crawl-delay: 1
65User-agent: Sogou Spider
66Disallow: /
67
68# http://www.exalead.com
69User-agent: *
70Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
71Crawl-delay: 1
72User-agent: Exabot
73Disallow: /
74
75# https://www.youdao.com/help/webmaster/spider/
76User-agent: *
77Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
78Crawl-delay: 1
79User-agent: YoudaoBot
80Disallow: /
81
82# https://megaindex.com/crawler
83User-agent: *
84Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
85Crawl-delay: 1
86User-agent: MegaIndex.ru
87Disallow: /
88
89User-agent: *
90Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
91Crawl-delay: 1
92User-agent: megaindex.com
93Disallow: /
94
95# https://napoveda.seznam.cz/en/fulltext-search/crawling/
96User-agent: *
97Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
98Crawl-delay: 1
99User-agent: SeznamBot
100Disallow: /
101
102# https://cliqz.com/company/cliqz-bot
103User-agent: *
104Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
105Crawl-delay: 1
106User-agent: Cliqzbot
107Disallow: /
108
109# https://www.gigablast.com/spider.html
110User-agent: *
111Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
112Crawl-delay: 1
113User-agent: Gigabot
114Disallow: /
115
116# http://openlinkprofiler.org/bot
117User-agent: *
118Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
119Crawl-delay: 1
120User-agent: spbot
121Disallow: /
122
123# https://developers.ezooms.com/ezooms/ezooms-bot
124User-agent: *
125Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
126Crawl-delay: 1
127User-agent: Ezooms
128Disallow: /
129
130# https://scoutjet.com/
131User-agent: *
132Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
133Crawl-delay: 1
134User-agent: ScoutJet
135Disallow: /
136
137# http://webmeup-crawler.com/
138User-agent: *
139Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
140Crawl-delay: 1
141User-agent: BLEXBot
142Disallow: /
143
144# Disallow: Sistrix
145User-agent: *
146Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
147Crawl-delay: 1
148User-agent: sistrix
149User-agent: *
150Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
151Crawl-delay: 1
152User-agent: SISTRIX Crawler
153User-agent: *
154Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
155Crawl-delay: 1
156User-agent: SISTRIX
157Disallow: /
158
159# Disallow: SEOkicks-Robot
160User-agent: *
161Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
162Crawl-delay: 1
163User-agent: SEOkicks
164Disallow:/
165
166User-agent: *
167Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
168Crawl-delay: 1
169User-agent: SEOkicks-Robot
170Disallow:/
171
172# http://www.searchmetrics.com
173User-agent: *
174Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
175Crawl-delay: 1
176User-agent: SearchmetricsBot
177Disallow: /
178
179# http://www.domaintools.com/webmasters/surveybot.php
180User-agent: *
181Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
182Crawl-delay: 1
183User-agent: SurveyBot
184Disallow: /
185
186# http://commoncrawl.org/faq/
187User-agent: *
188Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
189Crawl-delay: 1
190User-agent: CCBot
191Disallow: /
192
193# http://cliqz.com/company/cliqzbot
194User-agent: *
195Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
196Crawl-delay: 1
197User-agent: Cliqzbot
198Disallow: /
199
200# https://domainstats.com/pages/our-bot
201User-agent: *
202Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
203Crawl-delay: 1
204User-agent: DomainStatsBot
205Disallow: /
206
207# http://www.soso.com/help/webmasters.htm
208User-agent : Sosospider
209Disallow: /
210
211# https://teleport.dev/
212User-agent: *
213Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
214Crawl-delay: 1
215User-agent: Teleport
216Disallow: /
217
218# http://www.tenmax.com/teleportpro/
219User-agent: *
220Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
221Crawl-delay: 1
222User-agent: TeleportPro
223Disallow: /
224
225# https://www.httrack.com/
226User-agent: *
227Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
228Crawl-delay: 1
229User-agent: HTTrack
230Disallow: /
231
232########## END ##########