NeuralCrawl

Barracuda / robots.txt

← back to barracuda.com · change detected 2mo ago (2026-08-29T02:48:10Z) · created +0 −78

What changed

Diff old (2026-08-28T02:24:20Z) → new (2026-08-29T02:48:10Z)

@@ -1,6 +1,3 @@
1- User-agent: *
2- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
3- Crawl-delay: 1
41  User-agent: *
52 
63  # Prevent Crawling for Thank You Pages
@@ -45,131 +42,71 @@
4542  ########## PREVENT BAD BOTS TO CRAWL ##########
4643 
4744  # http://webmeup-crawler.com/
48- User-agent: *
49- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
50- Crawl-delay: 1
5145  User-agent: BLEXBot
5246  Disallow: /
5347 
5448  # https://cliqz.com/company/cliqz-bot
55- User-agent: *
56- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
57- Crawl-delay: 1
5849  User-agent: Cliqzbot
5950  Disallow: /
6051 
6152  # https://domainstats.com/pages/our-bot
62- User-agent: *
63- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
64- Crawl-delay: 1
6553  User-agent: DomainStatsBot
6654  Disallow: /
6755 
6856  # http://www.opensiteexplorer.org/dotbot
69- User-agent: *
70- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
71- Crawl-delay: 1
7257  User-agent: DotBot
73- User-agent: *
74- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
75- Crawl-delay: 1
7658  User-agent: dotbot
7759  Disallow: /
7860 
7961  # http://www.exalead.com
80- User-agent: *
81- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
82- Crawl-delay: 1
8362  User-agent: Exabot
8463  Disallow: /
8564 
8665  # https://developers.ezooms.com/ezooms/ezooms-bot
87- User-agent: *
88- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
89- Crawl-delay: 1
9066  User-agent: Ezooms
9167  Disallow: /
9268 
9369  # https://www.gigablast.com/spider.html
94- User-agent: *
95- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
96- Crawl-delay: 1
9770  User-agent: Gigabot
9871  Disallow: /
9972 
10073  # https://www.httrack.com/
101- User-agent: *
102- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
103- Crawl-delay: 1
10474  User-agent: HTTrack
10575  Disallow: /
10676 
10777  # https://megaindex.com/crawler
108- User-agent: *
109- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
110- Crawl-delay: 1
11178  User-agent: MegaIndex.ru
11279  Disallow: /
11380 
114- User-agent: *
115- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
116- Crawl-delay: 1
11781  User-agent: megaindex.com
11882  Disallow: /
11983 
12084  # https://scoutjet.com/
121- User-agent: *
122- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
123- Crawl-delay: 1
12485  User-agent: ScoutJet
12586  Disallow: /
12687 
12788  # http://www.searchmetrics.com
128- User-agent: *
129- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
130- Crawl-delay: 1
13189  User-agent: SearchmetricsBot
13290  Disallow: /
13391 
13492  # Disallow: SEOkicks-Robot
135- User-agent: *
136- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
137- Crawl-delay: 1
13893  User-agent: SEOkicks
13994  Disallow:/
14095 
141- User-agent: *
142- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
143- Crawl-delay: 1
14496  User-agent: SEOkicks-Robot
14597  Disallow:/
14698 
14799  # https://napoveda.seznam.cz/en/fulltext-search/crawling/
148- User-agent: *
149- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
150- Crawl-delay: 1
151100  User-agent: SeznamBot
152101  Disallow: /
153102 
154103  # Disallow: Sistrix
155- User-agent: *
156- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
157- Crawl-delay: 1
158104  User-agent: sistrix
159- User-agent: *
160- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
161- Crawl-delay: 1
162105  User-agent: SISTRIX Crawler
163- User-agent: *
164- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
165- Crawl-delay: 1
166106  User-agent: SISTRIX
167107  Disallow: /
168108 
169109  # http://help.sogou.com/homepage/search-engine/add-crawl.html
170- User-agent: *
171- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
172- Crawl-delay: 1
173110  User-agent: Sogou Spider
174111  Disallow: /
175112 
@@ -178,37 +115,22 @@
178115  Disallow: /
179116 
180117  # http://openlinkprofiler.org/bot
181- User-agent: *
182- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
183- Crawl-delay: 1
184118  User-agent: spbot
185119  Disallow: /
186120 
187121  # http://www.domaintools.com/webmasters/surveybot.php
188- User-agent: *
189- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
190- Crawl-delay: 1
191122  User-agent: SurveyBot
192123  Disallow: /
193124 
194125  # https://teleport.dev/
195- User-agent: *
196- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
197- Crawl-delay: 1
198126  User-agent: Teleport
199127  Disallow: /
200128 
201129  # http://www.tenmax.com/teleportpro/
202- User-agent: *
203- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
204- Crawl-delay: 1
205130  User-agent: TeleportPro
206131  Disallow: /
207132 
208133  # https://www.youdao.com/help/webmaster/spider/
209- User-agent: *
210- Disallow: /Sii8yP_qTEmmJiKsPYF7hZ21UiFM2l1ShyELGEXtZ9o=.html
211- Crawl-delay: 1
212134  User-agent: YoudaoBot
213135  Disallow: /
214136