NeuralCrawl

Barracuda / robots.txt

← back to barracuda.com · change detected 2mo ago (2026-08-28T02:24:20Z) · modified +133 −143

What changed

Diff old (2026-08-27T02:04:18Z) → new (2026-08-28T02:24:20Z)

@@ -1,5 +1,5 @@
11  User-agent: *
2- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
2+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
33  Crawl-delay: 1
44  User-agent: *
55 
@@ -39,187 +39,177 @@
3939  Disallow: */cache/*
4040  Disallow: */log/*
4141 
42-
4342  ########## SITEMAP FILES ##########
4443  Sitemap: https://www.barracuda.com/sitemap.xml
4544 
46-
47-
4845  ########## PREVENT BAD BOTS TO CRAWL ##########
4946 
50- # http://www.opensiteexplorer.org/dotbot
47+ # http://webmeup-crawler.com/
5148  User-agent: *
52- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
49+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
5350  Crawl-delay: 1
54- User-agent: DotBot
55- User-agent: *
56- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
57- Crawl-delay: 1
58- User-agent: dotbot
59- Disallow: /
60-
61- # http://help.sogou.com/homepage/search-engine/add-crawl.html
62- User-agent: *
63- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
64- Crawl-delay: 1
65- User-agent: Sogou Spider
66- Disallow: /
67-
68- # http://www.exalead.com
69- User-agent: *
70- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
71- Crawl-delay: 1
72- User-agent: Exabot
73- Disallow: /
74-
75- # https://www.youdao.com/help/webmaster/spider/
76- User-agent: *
77- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
78- Crawl-delay: 1
79- User-agent: YoudaoBot
80- Disallow: /
81-
82- # https://megaindex.com/crawler
83- User-agent: *
84- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
85- Crawl-delay: 1
86- User-agent: MegaIndex.ru
87- Disallow: /
88-
89- User-agent: *
90- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
91- Crawl-delay: 1
92- User-agent: megaindex.com
93- Disallow: /
94-
95- # https://napoveda.seznam.cz/en/fulltext-search/crawling/
96- User-agent: *
97- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
98- Crawl-delay: 1
99- User-agent: SeznamBot
51+ User-agent: BLEXBot
10052  Disallow: /
10153 
10254  # https://cliqz.com/company/cliqz-bot
10355  User-agent: *
104- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
105- Crawl-delay: 1
106- User-agent: Cliqzbot
107- Disallow: /
108-
109- # https://www.gigablast.com/spider.html
110- User-agent: *
111- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
112- Crawl-delay: 1
113- User-agent: Gigabot
114- Disallow: /
115-
116- # http://openlinkprofiler.org/bot
117- User-agent: *
118- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
119- Crawl-delay: 1
120- User-agent: spbot
121- Disallow: /
122-
123- # https://developers.ezooms.com/ezooms/ezooms-bot
124- User-agent: *
125- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
126- Crawl-delay: 1
127- User-agent: Ezooms
128- Disallow: /
129-
130- # https://scoutjet.com/
131- User-agent: *
132- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
133- Crawl-delay: 1
134- User-agent: ScoutJet
135- Disallow: /
136-
137- # http://webmeup-crawler.com/
138- User-agent: *
139- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
140- Crawl-delay: 1
141- User-agent: BLEXBot
142- Disallow: /
143-
144- # Disallow: Sistrix
145- User-agent: *
146- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
147- Crawl-delay: 1
148- User-agent: sistrix
149- User-agent: *
150- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
151- Crawl-delay: 1
152- User-agent: SISTRIX Crawler
153- User-agent: *
154- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
155- Crawl-delay: 1
156- User-agent: SISTRIX
157- Disallow: /
158-
159- # Disallow: SEOkicks-Robot
160- User-agent: *
161- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
162- Crawl-delay: 1
163- User-agent: SEOkicks
164- Disallow:/
165-
166- User-agent: *
167- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
168- Crawl-delay: 1
169- User-agent: SEOkicks-Robot
170- Disallow:/
171-
172- # http://www.searchmetrics.com
173- User-agent: *
174- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
175- Crawl-delay: 1
176- User-agent: SearchmetricsBot
177- Disallow: /
178-
179- # http://www.domaintools.com/webmasters/surveybot.php
180- User-agent: *
181- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
182- Crawl-delay: 1
183- User-agent: SurveyBot
184- Disallow: /
185-
186- # http://cliqz.com/company/cliqzbot
187- User-agent: *
188- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
56+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
18957  Crawl-delay: 1
19058  User-agent: Cliqzbot
19159  Disallow: /
19260 
19361  # https://domainstats.com/pages/our-bot
19462  User-agent: *
195- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
63+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
19664  Crawl-delay: 1
19765  User-agent: DomainStatsBot
66+ Disallow: /
67+
68+ # http://www.opensiteexplorer.org/dotbot
69+ User-agent: *
70+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
71+ Crawl-delay: 1
72+ User-agent: DotBot
73+ User-agent: *
74+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
75+ Crawl-delay: 1
76+ User-agent: dotbot
77+ Disallow: /
78+
79+ # http://www.exalead.com
80+ User-agent: *
81+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
82+ Crawl-delay: 1
83+ User-agent: Exabot
84+ Disallow: /
85+
86+ # https://developers.ezooms.com/ezooms/ezooms-bot
87+ User-agent: *
88+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
89+ Crawl-delay: 1
90+ User-agent: Ezooms
91+ Disallow: /
92+
93+ # https://www.gigablast.com/spider.html
94+ User-agent: *
95+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
96+ Crawl-delay: 1
97+ User-agent: Gigabot
98+ Disallow: /
99+
100+ # https://www.httrack.com/
101+ User-agent: *
102+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
103+ Crawl-delay: 1
104+ User-agent: HTTrack
105+ Disallow: /
106+
107+ # https://megaindex.com/crawler
108+ User-agent: *
109+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
110+ Crawl-delay: 1
111+ User-agent: MegaIndex.ru
112+ Disallow: /
113+
114+ User-agent: *
115+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
116+ Crawl-delay: 1
117+ User-agent: megaindex.com
118+ Disallow: /
119+
120+ # https://scoutjet.com/
121+ User-agent: *
122+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
123+ Crawl-delay: 1
124+ User-agent: ScoutJet
125+ Disallow: /
126+
127+ # http://www.searchmetrics.com
128+ User-agent: *
129+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
130+ Crawl-delay: 1
131+ User-agent: SearchmetricsBot
132+ Disallow: /
133+
134+ # Disallow: SEOkicks-Robot
135+ User-agent: *
136+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
137+ Crawl-delay: 1
138+ User-agent: SEOkicks
139+ Disallow:/
140+
141+ User-agent: *
142+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
143+ Crawl-delay: 1
144+ User-agent: SEOkicks-Robot
145+ Disallow:/
146+
147+ # https://napoveda.seznam.cz/en/fulltext-search/crawling/
148+ User-agent: *
149+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
150+ Crawl-delay: 1
151+ User-agent: SeznamBot
152+ Disallow: /
153+
154+ # Disallow: Sistrix
155+ User-agent: *
156+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
157+ Crawl-delay: 1
158+ User-agent: sistrix
159+ User-agent: *
160+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
161+ Crawl-delay: 1
162+ User-agent: SISTRIX Crawler
163+ User-agent: *
164+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
165+ Crawl-delay: 1
166+ User-agent: SISTRIX
167+ Disallow: /
168+
169+ # http://help.sogou.com/homepage/search-engine/add-crawl.html
170+ User-agent: *
171+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
172+ Crawl-delay: 1
173+ User-agent: Sogou Spider
198174  Disallow: /
199175 
200176  # http://www.soso.com/help/webmasters.htm
201177  User-agent : Sosospider
202178  Disallow: /
203179 
180+ # http://openlinkprofiler.org/bot
181+ User-agent: *
182+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
183+ Crawl-delay: 1
184+ User-agent: spbot
185+ Disallow: /
186+
187+ # http://www.domaintools.com/webmasters/surveybot.php
188+ User-agent: *
189+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
190+ Crawl-delay: 1
191+ User-agent: SurveyBot
192+ Disallow: /
193+
204194  # https://teleport.dev/
205195  User-agent: *
206- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
196+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
207197  Crawl-delay: 1
208198  User-agent: Teleport
209199  Disallow: /
210200 
211201  # http://www.tenmax.com/teleportpro/
212202  User-agent: *
213- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
203+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
214204  Crawl-delay: 1
215205  User-agent: TeleportPro
216206  Disallow: /
217207 
218- # https://www.httrack.com/
208+ # https://www.youdao.com/help/webmaster/spider/
219209  User-agent: *
220- Disallow: /GqS-MCUe_qfcMaNsUdAZhx3lvVU0SOEjrSLgx8DrQ9M=.html
210+ Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
221211  Crawl-delay: 1
222- User-agent: HTTrack
212+ User-agent: YoudaoBot
223213  Disallow: /
224214 
225215  ########## END ##########