# Robots.txt для cheeseit.ru — расширенная версия с явным разрешением AI-ботам
# Создано: 28 апреля 2026
# Заменяет существующий robots.txt; сохранить как /robots.txt

User-agent: *
Disallow: /ajax/
Disallow: /bitrix/
Disallow: /include/
Disallow: /personal/
Disallow: /local/
Disallow: /minify_bx/
Disallow: /kassa/
Disallow: /vendor/
Disallow: /functions/
Disallow: /404.php
Disallow: /search/
Disallow: /*?action=
Disallow: /*?login=
Disallow: /*?logout=
Disallow: /*?register=
Disallow: /*?auth=
Disallow: /*print=
Disallow: /*BASKET=
Disallow: /*COMPARE=
Disallow: /index.php

# AI-боты — явное разрешение (новое в этой версии)
User-agent: GPTBot
Allow: /
Disallow: /bitrix/
Disallow: /personal/
Disallow: /kassa/

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /
Disallow: /bitrix/
Disallow: /personal/

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

# Yandex
User-agent: Yandex
Disallow: /ajax/
Disallow: /bitrix/
Disallow: /include/
Disallow: /personal/
Disallow: /local/
Disallow: /kassa/
Disallow: /vendor/
Disallow: /404.php
Disallow: /search/
Disallow: /*?action=
Disallow: /*?login=
Disallow: /*?logout=
Disallow: /*?register=
Disallow: /*?auth=
Disallow: /*?password
Disallow: /*forgot
Disallow: /index.php
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term&utm_referrer&tpclid&calltouch_tm&utm_ya_campaign
Host: cheeseit.ru

# Googlebot
User-agent: Googlebot
Disallow: /ajax/
Disallow: /bitrix/
Disallow: /include/
Disallow: /personal/
Disallow: /local/
Disallow: /kassa/
Disallow: /vendor/
Disallow: /404.php
Disallow: /search/
Disallow: /*utm_*
Disallow: /*?action=
Disallow: /*?login=
Disallow: /*?logout=
Disallow: /*?register=
Disallow: /*?auth=
Disallow: /*?password
Disallow: /*forgot
Disallow: /index.php

# Sitemaps
Sitemap: https://cheeseit.ru/sitemap.xml