# robots.txt for Drupal (Optimized)
# Google and Ads bots - allow everything
User-agent: Mediapartners-Google
Disallow:
User-agent: AdsBot-Google
Disallow:
# Microsoft/Bing bots - allow everything
User-agent: bingbot
Disallow:
# General user agents - set crawl rate and block sensitive paths
User-agent: *
Crawl-delay: 10
# Allow access to CSS, JS, Images (Important for AdSense & Page Experience)
Allow: /core/*.css
Allow: /core/*.js
Allow: /core/*.png
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.gif
Allow: /modules/*.css
Allow: /modules/*.js
Allow: /modules/*.png
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.gif
Allow: /themes/*.css
Allow: /themes/*.js
Allow: /themes/*.png
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.gif
# Block backend and sensitive directories
Disallow: /admin/
Disallow: /user/
Disallow: /node/add/
Disallow: /node/preview/
Disallow: /comment/reply/
Disallow: /search/
Disallow: /filter/tips/
Disallow: /scripts/
Disallow: /includes/
Disallow: /profiles/
Disallow: /update.php
Disallow: /install.php
Disallow: /xmlrpc.php
# Block config and text files
Disallow: /*.txt$
Disallow: /CHANGELOG.txt
Disallow: /LICENSE.txt
Disallow: /UPGRADE.txt
Disallow: /INSTALL*
# Bot-specific crawl delays (these bots can overload)
User-agent: AhrefsBot
Crawl-delay: 30
User-agent: MJ12bot
Crawl-delay: 30
User-agent: SemrushBot
Crawl-delay: 30
User-agent: DotBot
Crawl-delay: 30
User-agent: BLEXBot
Crawl-delay: 30
robots.txt