# GENERAL BOT RULES (limit load)
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10
# BLOCK ABUSIVE BOTS COMPLETELY
User-agent: AhrefsBot
Disallow: /
User-agent: YisouSpider
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SeekportBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: DotBot
Disallow: /
# GOOGLE AND BING SPECIFIC CRAWL RULES
User-agent: Googlebot
Disallow:
Crawl-delay: 50
User-agent: Bingbot
Disallow:
Crawl-delay: 50
robots.txt