# =========================================================
# AI agents: user-initiated access
# These agents fetch pages on behalf of a user or site owner.
# Public site functions, query parameters, search, sorting,
# favorites and basket are intentionally available to them.
# =========================================================
User-agent: Claude-User
User-agent: ChatGPT-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: Amzn-User
User-agent: Google-CloudVertexBot
Content-Signal: search=yes, ai-input=yes
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms/
# Explicit compatibility rules for user-initiated AI fetchers.
# Restrictions in later crawler groups are not intended for the user agents above.
Allow: /catalog/
Allow: /catalog/*?*
Allow: /search/
Allow: /search/?search=*
Allow: /personal/basket/
Allow: /favorits/
Disallow: /bitrix
Disallow: /*index*.php
Disallow: /login
Disallow: /cr_on
Disallow: /about/personal_data
Disallow: /about/user_agreement/
Disallow: /servis/*
Disallow: /search_ajax/*
Allow: /


# =========================================================
# AI search / indexing crawlers
# Keep the previous SEO crawling restrictions.
# =========================================================
User-agent: Claude-SearchBot
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: MistralAI-Index
User-agent: Meta-WebIndexer
User-agent: Amzn-SearchBot
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms/
Disallow: /bitrix
Disallow: /*index*.php
Disallow: /search
Disallow: /login
Disallow: /personal
Disallow: /cr_on
Disallow: /favorits
Disallow: /about/personal_data
Disallow: /about/user_agreement/
Disallow: /*?sorts
Disallow: /*?sort=price_min
Disallow: /*?sort=price_max
Disallow: /*?*
Disallow: /servis/*
Disallow: /search/*
Disallow: /search_ajax/*
Allow: /*?page=*
Allow: /bitrix/js
Allow: /bitrix/fonts/*
Allow: /bitrix/components
Allow: /assets/image
Allow: */assets/js/*
Allow: */assets/*
Allow: */assets/css/*
Allow: /*.js
Allow: /*.css
Allow: /*.jpg
Allow: /*.gif
Allow: /*.png
Allow: /*.webp
Allow: /*.js*
Allow: /*.css*
Allow: */*.js
Allow: */*.css
Allow: */*.js*
Allow: */*.css*
Allow: /upload/iblock/*.jpg
Allow: /upload/iblock/*.png
Allow: /upload/iblock/*.gif
Allow: /upload/iblock/*.webp
Allow: /*/*/*/*/*/*/*.css
Allow: /*/*/*/*/*/*/*.js


# =========================================================
# Default policy for all other crawlers
# Preserved from the previous robots.txt.
# =========================================================
User-agent: *
Content-Signal: search=yes, ai-input=yes
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms/
Clean-param: clid
Clean-param: back_url_admin
Clean-param: clear_cache
Clean-param: show_include_exec_time
Clean-param: show_page_exec_time
Clean-param: show_sql_stat
Clean-param: srsltid

Disallow: /bitrix
Disallow: /*index*.php
Disallow: /search
Disallow: /login
Disallow: /personal
Disallow: /cr_on
Disallow: /favorits
Disallow: /about/personal_data
Disallow: /about/user_agreement/
Disallow: /*?sorts
Disallow: /*?sort=price_min
Disallow: /*?sort=price_max
Disallow: /*?*
Disallow: /servis/*
Disallow: /search/*
Disallow: /search_ajax/*

Allow: /*?page=*
Allow: /bitrix/js
Allow: /bitrix/fonts/*
Allow: /bitrix/components
Allow: /assets/image
Allow: */assets/js/*
Allow: */assets/*
Allow: */assets/css/*
Allow: /*.js
Allow: /*.css
Allow: /*.jpg
Allow: /*.gif
Allow: /*.png
Allow: /*.webp
Allow: /*.js*
Allow: /*.css*
Allow: */*.js
Allow: */*.css
Allow: */*.js*
Allow: */*.css*
Allow: /upload/iblock/*.jpg
Allow: /upload/iblock/*.png
Allow: /upload/iblock/*.gif
Allow: /upload/iblock/*.webp
Allow: /*/*/*/*/*/*/*.css
Allow: /*/*/*/*/*/*/*.js

Host: https://vesremont.com/
Sitemap: https://vesremont.com/sitemaps/sitemap_page.xml