# Robot rules for rent.dockdive.com
User-agent: *
Allow: /
# Block certain crawling patterns
Disallow: /auth/
Disallow: /dashboard/
Disallow: /*?* # Block query parameters
# Allow specific crawlers to access all public content
User-agent: Googlebot
Allow: /
Disallow: /auth/
Disallow: /dashboard/
User-agent: Bingbot
Allow: /
Disallow: /auth/
Disallow: /dashboard/
# Crawl-delay directive for non-major search engines
User-agent: *
Crawl-delay: 10
# Sitemap location
Sitemap: https://rent.dockdive.com/sitemap.xml
robots.txt