# AI crawlers (decision: docs/marketing/decisions/2026-09-26-ai-crawlers.md).
# Answer engines and assistants fetching a page for a person — OAI-SearchBot,
# ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User —
# and the training crawlers of assistants our customers use — GPTBot, ClaudeBot,
# Google-Extended, Applebot-Extended — follow the "*" group below, like Google.
# Scrapers that send no one back are refused the whole site.
User-agent: Bytespider
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: cohere-training-data-crawler
Disallow: /

User-agent: *
# Private or transactional — never indexable.
Disallow: /checkout
Disallow: /cart/
Disallow: /account/
Disallow: /admin
Disallow: /ai/
Disallow: /go/
Disallow: /pay/
Disallow: /auth/
# Sign-in with a return address (/login?next=…): the same noindex form once per
# page that links to it. /login itself stays open (SEO-012a, 2026-10-17).
Disallow: /login?
Disallow: /price-estimator/
# Endpoints that page scripts call (chat widget, APIs) — not pages.
Disallow: /chat/
Disallow: /api/
# Short links: counted redirects to pages that are crawlable on their own.
Disallow: /l/
# …except the specs the /docs pages render from.
Allow: /api/openapi.json
Allow: /api/gateway/openapi.json
Allow: /api/ai/openapi.json
Allow: /api/fx/openapi.json
Allow: /api/markets/openapi.json
# Search, sort and filter variants of listings; ?page= stays crawlable.
Disallow: /*?q=
Disallow: /*&q=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?delivery=
Disallow: /*&delivery=
Disallow: /*?min_price=
Disallow: /*&min_price=
Disallow: /*?max_price=
Disallow: /*&max_price=
Allow: /

Sitemap: https://netarz.ir/sitemap.xml