User-agent: AHC/2.1
User-Agent: Amazonbot
User-Agent: Applebot-Extended
User-agent: Bytespider
User-agent: ChatGLM
User-agent: ClaudeBot
User-Agent: Facebookbot
User-Agent: Google-CloudVertexBot
User-Agent: Google-Extended
User-agent: GPTBot
User-Agent: Meta-ExternalAgent
User-Agent: Meta-ExternalFetcher
User-agent: PetalBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: Thinkbot
Disallow: /
DisallowAITraining: /
User-agent: *
Crawl-delay: 10
Disallow: /activities/
Disallow: /additional_information
Disallow: /ads/
Disallow: /best_of_web/*/flip
Disallow: /best_of_web/*/think
Disallow: /best_of_web/*/discuss
Disallow: /lessons/*/flip
Disallow: /lessons/*/think
Disallow: /lessons/*/discuss
Disallow: /lessons?category=*&
Disallow: /lessons?*student_level[]=
Disallow: /lessons?*content_type[]=
Disallow: /lessons?*subtitles[]=
Disallow: /lessons?*duration=
Disallow: /on/
Disallow: /*/*/discussions/
Disallow: /*/*/open/
Disallow: /*/*/guided/
Disallow: /search/
Disallow: /u/
Disallow: /users/new
Disallow: /videos
Sitemap: https://teded.tedcdn.com/sitemaps/sitemap.xml.gz
robots.txt