User-agent: SiteAuditBot
Crawl-delay:1
Allow: /
User-agent: Semrushbot-SI
Allow: /
# Default rules for all other bots
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /trackback/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /comments/
Disallow: /category/*/feed/
Allow: /wp-content/uploads/
# Sitemap location
Sitemap: https://www.prodigitalweb.com/sitemap.xml
robots.txt