# ===================================================================
# Optimized Production Robots.txt - eCommerceFastlane.com
# Verified for 2026 Standards - AI Visibility & SEO Efficiency
# ===================================================================
# -------------------------------------------------------------------
# GROUP 1: SEARCH ENGINES & PREMIUM AI (Maximum Visibility)
# -------------------------------------------------------------------
# Includes Google, Bing, OpenAI (ChatGPT), Anthropic (Claude),
# Perplexity, Apple, and emerging AI search engines.
# All efficiency rules (blocking search results, feeds, tracking
# parameters) are included here to ensure proper inheritance.
# -------------------------------------------------------------------
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Bingbot
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
User-agent: ClaudeBot
User-agent: claude-web
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: facebookexternalhit
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: YouBot
User-agent: PhindBot
User-agent: ExaBot
User-agent: AndiBot
User-agent: DuckAssistBot
User-agent: cohere-ai
Allow: /
Allow: /wp-json/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /readme.html
Disallow: /license.txt
# Duplicate Content & Efficiency Blocks (CRITICAL)
Disallow: /?s=
Disallow: /*?s=
Disallow: /*?replytocom
Disallow: /trackback/
Disallow: /*?utm_
Disallow: /*?share=
Disallow: /feed/
Disallow: /comments/feed/
# -------------------------------------------------------------------
# GROUP 2: COMMON CRAWL & DATASETS
# -------------------------------------------------------------------
# Large-scale crawlers used for AI training datasets.
# High bandwidth consumption - monitor server load.
# If performance issues arise, change Allow to Disallow.
User-agent: CCBot
User-agent: Diffbot
Allow: /
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /feed/
# -------------------------------------------------------------------
# GROUP 3: SEO TOOLS (Rate Limited)
# -------------------------------------------------------------------
# Note: Crawl-delay respected by Bing/Yandex, ignored by Google.
# wp-json blocked to limit competitor intelligence gathering.
User-agent: AhrefsBot
User-agent: SemrushBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /?s=
User-agent: MJ12bot
Crawl-delay: 20
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /?s=
# -------------------------------------------------------------------
# GROUP 4: DEFAULT RULES (All Other Crawlers)
# -------------------------------------------------------------------
# Fallback rules for any crawler not explicitly listed above.
# Note: Crawl-delay ignored by Google, respected by Bing/Yandex.
User-agent: *
Crawl-delay: 1
# Allow critical assets for proper page rendering
Allow: /*.css$
Allow: /*.js$
Allow: /wp-content/uploads/
# Block Admin & Core WordPress Files
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
# Block Duplicate Content & Efficiency Killers
Disallow: /?s=
Disallow: /*?s=
Disallow: /*?replytocom
Disallow: /trackback/
Disallow: /*?utm_
Disallow: /*?share=
Disallow: /feed/
Disallow: /comments/feed/
# -------------------------------------------------------------------
# SITEMAP
# -------------------------------------------------------------------
Sitemap: https://ecommercefastlane.com/sitemap_index.xml
robots.txt