# === Facebook / Meta crawlers ===
User-agent: facebookexternalhit
Disallow: /

User-agent: Facebot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# === Amazon crawlers ===
User-agent: Amazonbot
Disallow: /

# --- BLOCK AI & USELESS BOTS (Reduce server load) ---
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Bytespider
Disallow: /

# --- THROTTLE BING (Prevent DB overload) ---
User-agent: bingbot
Crawl-delay: 10

# --- RULES FOR ALL BOTS (CS-Cart Specifics) ---
User-agent: *

# 1. System folders and files (Security + junk)
Disallow: /app/
Disallow: /design/
Disallow: /store_closed.html
Disallow: /*features_hash=$
Disallow: /*-page-*
Disallow: /*page=*
Allow: /var/cache/misc/*.css
Allow: /var/cache/misc/*.js
Allow: /images/thumbnails/*/feature_variant/*
Allow: /images/thumbnails/*/detailed/*
Allow: /js/*.js
Allow: /design/themes/*.jpg
Allow: /images/*.jpg
Allow: /design/themes/*.gif
Allow: /design/themes/*.ttf
Allow: /design/themes/*.woff
Allow: /design/themes/*.svg
Allow: /design/themes/*.png
Disallow: /?gclid=

# 2. Checkout, profile, cart (Not needed in index)
Disallow: /checkout
Disallow: /cart
Disallow: /my-account
Disallow: /login
Disallow: /wishlist
Disallow: /compare

# 3. TECHNICAL (MySQL performance killers)
# Block sorting, pagination limits, currency/layout switching - these create duplicate pages and heavy queries
Disallow: /*?sort_by=
Disallow: /*&sort_by=
Disallow: /*?items_per_page=
Disallow: /*&items_per_page=
Disallow: /*?currency=
Disallow: /*&currency=
Disallow: /*?layout=
Disallow: /*&layout=

# 4. FILTERS (features_hash) - Major performance impact
# Bots combining multiple filters can overload the database with heavy queries.
# Highly recommended to keep this disallowed unless you have a specific SEO strategy for filter pages.
Disallow: /*features_hash=

# 5. Site Search (Generates heavy queries)
Disallow: /*dispatch=products.search
Disallow: /*q=

Sitemap: https://sad-ogorod.in.ua/sitemap.xml