# Allow all search engines to crawl the main content
User-agent: *
Disallow: /wp-admin/ # Block admin area
Disallow: /category/ # Block Category Directory
Disallow: /tag/ # Block tag archives
Disallow: /1000/ # Block specific directory
Disallow: /2021/ # Block specific directory
Disallow: /2022/ # Block specific directory
Disallow: /2023/ # Block specific directory
Disallow: /2024/ # Block specific directory
Disallow: /2/ # Block specific directory
Disallow: /feed/ # Block RSS feeds
Disallow: /wp-content/ # Block RSS wp-content directory
Disallow: /wp-json/ # Block RSS wp-json directory
Allow: *.css # Allow CSS files to ensure proper rendering
Allow: *.js # Allow JavaScript files to ensure proper rendering
#not sure - Disallow: /itadon-faqs/
# Specific rules for Googlebot and related bots
User-agent: Googlebot
Allow: /
Allow: *.js
Allow: *.css
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Mobile
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: Mediapartners-Google
Allow: /
User-agent: AdsBot-Google
Allow: /
# Specific rules for other major search engines
User-agent: Bingbot
Allow: /
User-agent: Slurp
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Baiduspider
Allow: /
User-agent: Baiduspider-image
Allow: /
User-agent: YandexBot
Allow: /
User-agent: facebot
Allow: /
User-agent: Applebot
Allow: /
User-agent: MSNBot
Allow: /
User-agent: AMZNKAssocBot
Allow: /
User-agent: ia_archiver
Allow: /
# Disallow less important crawlers to save server resources
User-agent: dotbot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: barkrowler
Disallow: /
# Crawl delay for specific bots that may overload the server
User-agent: ImagesiftBot
Crawl-delay: 8
Sitemap: https://itadon.com/sitemap_index.xml
robots.txt