sudo nano /var/www/html/webpronews.com/public_html/robots.txt
```
```
# WebProNews.com Robots.txt
# Last updated: 2026-01-12
# ===========================================
# DEFAULT RULES FOR ALL CRAWLERS
# ===========================================
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /tag/*/feed/
Disallow: /category/*/feed/
Disallow: /author/*/feed/
Disallow: /?p=
Disallow: /*?replytocom=
Disallow: /*?preview=
Disallow: /page/
Disallow: /*/*/page/
Disallow: /tag/*/page/
Disallow: /category/*/page/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Crawl-delay: 1
# ===========================================
# GOOGLE - No crawl delay (they ignore it anyway)
# ===========================================
User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=
Disallow: /*?replytocom=
Disallow: /*?preview=
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /*.css
Allow: /*.js
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /wp-admin/
# ===========================================
# BING - Aggressive, needs throttling
# ===========================================
User-agent: bingbot
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=
Disallow: /*?replytocom=
Disallow: /tag/*/page/
Disallow: /category/*/page/
Disallow: /page/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Crawl-delay: 10
User-agent: msnbot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /?s=
# ===========================================
# YAHOO - Very aggressive
# ===========================================
User-agent: Slurp
Disallow: /wp-admin/
Disallow: /?s=
Disallow: /search/
Disallow: /tag/*/page/
Disallow: /category/*/page/
Disallow: /page/
Crawl-delay: 10
# ===========================================
# OTHER SEARCH ENGINES
# ===========================================
User-agent: DuckDuckBot
Disallow: /wp-admin/
Disallow: /?s=
Allow: /wp-content/uploads/
Crawl-delay: 5
User-agent: Baiduspider
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 10
User-agent: YandexBot
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 10
# ===========================================
# AI CRAWLERS - Allow but throttle
# ===========================================
User-agent: GPTBot
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 5
User-agent: ChatGPT-User
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 5
User-agent: anthropic-ai
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 5
User-agent: Claude-Web
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 5
User-agent: OAI-SearchBot
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 5
User-agent: PerplexityBot
Disallow: /wp-admin/
Disallow: /?s=
Crawl-delay: 5
# ===========================================
# BLOCK BAD BOTS ENTIRELY
# ===========================================
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: SearchAtlas
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: serpstatbot
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: CCBot
Disallow: /
# ===========================================
# SITEMAP LOCATION
# ===========================================
Sitemap: https://www.webpronews.com/sitemap_index.xml
robots.txt