User-agent: * # applies to all robots
Disallow: /*/feed/$
Disallow: /?nonamp=1/
Disallow: /?amp=1
Disallow: /?noamp=mobile
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# Adding Multiple Sitemaps
Allow: Sitemap: https://www.technologyford.com/sitemap_index.xml
Allow: Sitemap: https://www.technologyford.com/post-sitemap.xml
Allow: Sitemap: https://www.technologyford.com/page-sitemap.xml
Allow: Sitemap: https://www.technologyford.com/category-sitemap.xml
Allow: Sitemap: https://www.technologyford.com/author-sitemap.xml
# Allowed Good User Agents for better Crawl
User-agent: Googlebot
Disallow:
Allow: /*
User-agent: Mediapartners-Google*
Disallow:
Allow: /*
User-agent: Boomtrain-Content-Bot*
Disallow:
Allow: /*
User-agent: Googlebot-Image
Disallow:
Allow: /*
User-agent: Adsbot-Google
Disallow:
Allow: /*
User-agent: Googlebot-News
Disallow:
Allow: /*
User-agent: Bingbot
Disallow:
User-agent: MSNBot
Disallow:
User-agent: Slurp
Disallow:
User-agent: DuckDuckBot
Disallow:
User-agent: Baiduspider
Disallow:
User-agent: YandexBot
Disallow:
User-agent: ia_archiver
Disallow:
User-agent: Teoma
Disallow:
User-agent: Rogerbot
Disallow:
User-agent: Rogerbot/1.2
Disallow:
User-agent: DotBot
Disallow:
User-agent: DotBot/1.1
Disallow:
User-agent: AhrefsBot
Disallow:
Allow: /*
User-agent: MJ12bot
Disallow:
Allow: /*
User-agent: SemrushBot
Disallow:
Allow: /*
User-agent: NinjaBot
Disallow:
User-agent: Facebot
Disallow:
User-agent: Twitterbot
Disallow:
User-agent: LinkedInBot
Disallow:
robots.txt