# www.robotstxt.org/
# Allow crawling of all content
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://voclai.com/sitemap.xml

# Optimize crawling - don't crawl assets and other non-essential directories
User-agent: *
Disallow: /assets/
Disallow: /static/media/
Disallow: /static/css/
Disallow: /static/js/

# Allow important language paths to be indexed
User-agent: *
Allow: /
Allow: /ru/
Allow: /de/
Allow: /fr/
Allow: /es/
Allow: /ko/
Allow: /ja/
Allow: /pt/
Allow: /ar/
Allow: /zh/