# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /
User-Agent: *
Disallow: /ajax/
Disallow: /api/
Disallow: /search
Disallow: /results/
Disallow: /r/
Disallow: /episodes/*/embed
Disallow: /*_openstat*
Disallow: /*gclid=*
Disallow: /*utm_source=*
Disallow: *utm_medium=*
Disallow: *utm_campaign=*
Disallow: *utm_term=*
Disallow: *fbclid=*
Crawl-delay: 2
Clean-param: src&state&result /
Clean-param: rubric /mag

Host: https://arzamas.academy

Sitemap: https://arzamas.academy/shared/sitemap.xml.gz