NeuralCrawl

🇺🇸 Barracuda

barracuda.com · Cybersecurity · rank #34 · Cybersecurity · live robots.txt ↗

AI crawler access (latest snapshot, 5h ago)

⛔blocked restricted ✅allowed faded = inherited from the * wildcard group

✅GPTBot
✅ChatGPT-User
✅OAI-SearchBot
✅ClaudeBot
✅Claude-User
✅Claude-SearchBot
✅anthropic-ai
✅Claude-Web
⛔CCBot
✅Google-Extended
✅Applebot-Extended
✅PerplexityBot
✅Perplexity-User
✅Bytespider
✅Amazonbot
✅FacebookBot
✅meta-externalagent
✅meta-externalfetcher
✅cohere-ai
✅AI2Bot
✅Diffbot
✅omgili
✅YouBot
✅DuckAssistBot
✅MistralAI-User
✅PanguBot
✅Timpibot

Current robots.txt 5217 bytes · sha256 d646e35db798 · raw 

User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: *

# Prevent Crawling for Thank You Pages
Disallow: */*LP-OnDemand*
Disallow: */*LP-ThankYou*
Disallow: */*thankyou*
Disallow: */*thank-you*
Disallow: */thankyou/*
Disallow: */thankyou*
Disallow: */thankYou*

# Prevent Crawling for Login & Register
Disallow: */login/*
Disallow: */register/*
Disallow: */customers/s/login/*
Disallow: */auth/login/*

# Prevent Error Crawling for Templates
Disallow: /error/

# Prevent Crawling for Captcha
Disallow: */captcha/*

# Prevent Crawling for Admin
Disallow: */admin/*
Disallow: */administrator/*

# Prevent Crawling for User
Disallow: */user/*
Disallow: */users/*

# Prevent Crawling for CGI
Disallow: /cgi-bin/

# Prevent Crawling for cache and log
Disallow: */cache/*
Disallow: */log/*


########## SITEMAP FILES ##########
Sitemap: https://www.barracuda.com/sitemap.xml



########## PREVENT BAD BOTS TO CRAWL ##########

# http://www.opensiteexplorer.org/dotbot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: DotBot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: dotbot
Disallow: /

# http://help.sogou.com/homepage/search-engine/add-crawl.html
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Sogou Spider
Disallow: /

# http://www.exalead.com
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Exabot
Disallow: /

# https://www.youdao.com/help/webmaster/spider/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: YoudaoBot
Disallow: /

# https://megaindex.com/crawler
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: MegaIndex.ru
Disallow: /

User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: megaindex.com
Disallow: /

# https://napoveda.seznam.cz/en/fulltext-search/crawling/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SeznamBot
Disallow: /

# https://cliqz.com/company/cliqz-bot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Cliqzbot
Disallow: /

# https://www.gigablast.com/spider.html
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Gigabot
Disallow: /

# http://openlinkprofiler.org/bot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: spbot
Disallow: /

# https://developers.ezooms.com/ezooms/ezooms-bot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Ezooms
Disallow: /

# https://scoutjet.com/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: ScoutJet
Disallow: /

# http://webmeup-crawler.com/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: BLEXBot
Disallow: /

# Disallow: Sistrix
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: sistrix
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SISTRIX Crawler
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SISTRIX
Disallow: /

# Disallow: SEOkicks-Robot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SEOkicks
Disallow:/

User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SEOkicks-Robot
Disallow:/

# http://www.searchmetrics.com
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SearchmetricsBot
Disallow: /

# http://www.domaintools.com/webmasters/surveybot.php
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: SurveyBot
Disallow: /

# http://commoncrawl.org/faq/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: CCBot
Disallow: /

# http://cliqz.com/company/cliqzbot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Cliqzbot
Disallow: /

# https://domainstats.com/pages/our-bot
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: DomainStatsBot
Disallow: /

# http://www.soso.com/help/webmasters.htm
User-agent: Sosospider
Disallow: /

# https://teleport.dev/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: Teleport
Disallow: /

# http://www.tenmax.com/teleportpro/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: TeleportPro
Disallow: /

# https://www.httrack.com/
User-agent: *
Disallow: /H1Rj9-8bvN3I9mQlIPVT5Cm31oOG5HamhkmPLNbEbyE=.html
Crawl-delay: 1
User-agent: HTTrack
Disallow: /

########## END ##########

Change history

  1. initial snapshot
    • First snapshot of robots.txt archived