User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: AHC/2.1
Disallow: /
User-agent: GPTBot/1.1
Disallow: /
User-agent: *
Crawl-delay: 10
Disallow: /activities/
Disallow: /ads/
Disallow: /best_of_web/*/flip
Disallow: /best_of_web/*/think
Disallow: /best_of_web/*/discuss
Disallow: /lessons/*/flip
Disallow: /lessons/*/think
Disallow: /lessons/*/discuss
Disallow: /on/
Disallow: /*/*/discussions/
Disallow: /*/*/open/
Disallow: /*/*/guided/
Disallow: /users/new
Disallow: /videos/
Disallow: /search
Disallow: /lessons/?category=*&*
Disallow: /lessons/?student_level=*&*
Disallow: /lessons?_*&page=*
Disallow: /additional_information
Disallow: /u/
Sitemap: https://teded.tedcdn.com/sitemaps/sitemap.xml.gz
robots.txt