# CodeGraph Landing Page robots.txt # https://codegraph.ru/ User-agent: GPTBot Disallow: / User-agent: OAI-SearchBot Allow: / User-agent: ClaudeBot Allow: / User-agent: PerplexityBot Allow: / User-agent: Google-Extended Allow: / User-agent: * Allow: / # Yandex duplicate URL cleanup for insignificant query params Clean-param: ybaip Clean-param: etext # Sitemap location Sitemap: https://codegraph.ru/sitemap.xml # Crawl-delay for polite crawling Crawl-delay: 1 # Disallow internal/service paths Disallow: /admin/ Disallow: /api/ Disallow: /templates/ Disallow: /.claude/ Disallow: /*.json$ # Allow all assets Allow: /css/ Allow: /js/ Allow: /assets/ Allow: /llms.txt Allow: /llms-full.txt