# Dark Visitors robots.txt from https://darkvisitors.com/robots-txt-builder
# AI Data Scraper
# https://darkvisitors.com/agents/ai2bot
User-agent: AI2Bot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/applebot-extended
User-agent: Applebot-Extended
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/bytespider
User-agent: Bytespider
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/ccbot
User-agent: CCBot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/claudebot
User-agent: ClaudeBot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/diffbot
User-agent: Diffbot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/cohere-training-data-crawler
User-agent: cohere-training-data-crawler
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/cotoyogi
User-agent: Cotoyogi
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/datenbank-crawler
User-agent: Datenbank Crawler
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/facebookbot
User-agent: FacebookBot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/factset-spyderbot
User-agent: Factset_spyderbot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/google-extended
User-agent: Google-Extended
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/gptbot
User-agent: GPTBot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/icc-crawler
User-agent: ICC-Crawler
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/kangaroo-bot
User-agent: Kangaroo Bot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/meta-externalagent
User-agent: Meta-ExternalAgent
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/netestate-imprint-crawler
User-agent: netEstate Imprint Crawler
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/omgili
User-agent: omgili
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/pangubot
User-agent: PanguBot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/petalbot
User-agent: PetalBot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/semrushbot-ocob
User-agent: SemrushBot-OCOB
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/timpibot
User-agent: Timpibot
Disallow: /
# AI Data Scraper
# https://darkvisitors.com/agents/webzio-extended
User-agent: Webzio-Extended
Disallow: /
# All other crawlers
User-agent: *
Disallow: /*/by/* # sorting options
Disallow: /*/popular/this/* # popularity sorting options
Disallow: /*/on/* # availability options
Disallow: /*/tag/* # Members? tag lists
Disallow: /*/genre/* # Films by genre
Disallow: /*/country/* # Films by country
Disallow: /*/language/* # Films by language
Disallow: /*/decade/* # Films by decade
Disallow: /films/year/* # Films by year
Disallow: /films/*/year/* # Films by year
Disallow: /films/*/size/large/* # Films with large posters (and therefore stats)
Disallow: /*/friends/* # stuff grouped for users' friends
robots.txt