# Dark Visitors robots.txt from https://darkvisitors.com/robots-txt-builder

# AI Data Scraper
# https://darkvisitors.com/agents/ai2bot

User-agent: AI2Bot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/applebot-extended

User-agent: Applebot-Extended
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/bytespider

User-agent: Bytespider
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/ccbot

User-agent: CCBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/claudebot

User-agent: ClaudeBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/diffbot

User-agent: Diffbot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/cohere-training-data-crawler

User-agent: cohere-training-data-crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/cotoyogi

User-agent: Cotoyogi
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/datenbank-crawler

User-agent: Datenbank Crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/facebookbot

User-agent: FacebookBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/factset-spyderbot

User-agent: Factset_spyderbot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/google-extended

User-agent: Google-Extended
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/gptbot

User-agent: GPTBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/icc-crawler

User-agent: ICC-Crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/kangaroo-bot

User-agent: Kangaroo Bot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/meta-externalagent

User-agent: Meta-ExternalAgent
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/netestate-imprint-crawler

User-agent: netEstate Imprint Crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/omgili

User-agent: omgili
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/pangubot

User-agent: PanguBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/petalbot

User-agent: PetalBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/semrushbot-ocob

User-agent: SemrushBot-OCOB
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/timpibot

User-agent: Timpibot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/webzio-extended

User-agent: Webzio-Extended
Disallow: /

# Indexes content for Algolia search engine - which 
# https://radar.cloudflare.com/bots/directory/algolia

User-Agent: Algolia Crawler
Disallow: /

# All other crawlers
User-agent: *
Disallow: /*/by/*                     # sorting options
Disallow: /*/popular/this/*           # popularity sorting options
Disallow: /*/on/*                     # availability options
Disallow: /*/tag/*                    # Members? tag lists
Disallow: /*/genre/*                  # Films by genre
Disallow: /*/country/*                # Films by country
Disallow: /*/language/*               # Films by language
Disallow: /*/decade/*                 # Films by decade
Disallow: /films/year/*               # Films by year
Disallow: /films/*/year/*             # Films by year
Disallow: /films/*/size/large/*       # Films with large posters (and therefore stats)
Disallow: /*/friends/*                # stuff grouped for users' friends