# ==================================================
# Global rules – WordPress crawl budget protection
# ==================================================
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?replytocom=
Disallow: /wp-json/
Allow: /wp-json/wp/v2/
# ==================================================
# Google (Search, Discover, Images, Video, News)
# ==================================================
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: Googlebot-News
Allow: /
# ==================================================
# Bing / Microsoft
# ==================================================
User-agent: Bingbot
Allow: /
User-agent: MSNBot
Allow: /
# ==================================================
# Yandex
# ==================================================
User-agent: Yandex
Allow: /
# ==================================================
# AI & dataset crawlers (allowed for citations)
# ==================================================
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: CCBot
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
# ==================================================
# SEO & analytics tools (allowed + throttled)
# ==================================================
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10
User-agent: SemrushBot
Allow: /
Crawl-delay: 10
User-agent: SemrushBot-SA
Allow: /
Crawl-delay: 10
User-agent: MJ12bot
Allow: /
Crawl-delay: 10
User-agent: DotBot
Allow: /
Crawl-delay: 10
User-agent: BLEXBot
Allow: /
Crawl-delay: 10
User-agent: Rogerbot
Allow: /
Crawl-delay: 10
User-agent: SeobilityBot
Allow: /
Crawl-delay: 10
# ==================================================
# Sitemap
# ==================================================
Sitemap: https://barrazacarlos.com/sitemap_index.xml
robots.txt