User-agent: *

# ── Public pages — crawl everything ──────────────────────────────
Allow: /
Allow: /about
Allow: /pricing
Allow: /email-inspector
Allow: /blog
Allow: /blog/
Allow: /ai
Allow: /anubis-engine

# ── Platform pages — PUBLIC, must be indexed ─────────────────────
Allow: /overview
Allow: /threats
Allow: /threats/
Allow: /actors
Allow: /ioc
Allow: /timeline
Allow: /intel
Allow: /grc
Allow: /calculator
Allow: /news
Allow: /help

# ── Auth-required pages — no value for crawlers ──────────────────
Disallow: /sandbox
Disallow: /profile
Disallow: /auth
Disallow: /auth/

# ── Backend API — never crawl ─────────────────────────────────────
Disallow: /api/

# ── Crawler-specific rules ────────────────────────────────────────
User-agent: GPTBot
Allow: /threats/
Allow: /anubis-engine
Allow: /about
Allow: /pricing
Disallow: /api/

User-agent: Anthropic-AI
Allow: /threats/
Allow: /anubis-engine
Allow: /about
Disallow: /api/

User-agent: PerplexityBot
Allow: /threats/
Allow: /anubis-engine
Disallow: /api/

Sitemap: https://riskwatchpro.online/sitemap.xml