# Default: all crawlers welcome except on sensitive routes
User-agent: *
Allow: /
Allow: /assets/
Allow: /_next/static/
Allow: /_next/image/
Allow: /static/

# Disallow sensitive routes
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /cdn-cgi/

# Disallow query parameters on main tools listing page only
# Individual tool pages (/tools/[tool-name]) handle canonicals automatically
Disallow: /tools?*

# Content Signals (contentsignals.org): this site's content may be used for
# search, as AI input (grounding, answers, citations) and for AI training.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# --- AI assistant and answer-engine crawlers: explicitly allowed (docs/32) ---
# Bingbot is the retrieval layer behind ChatGPT search and Copilot; it is covered
# by the default group above. The groups below repeat the default for crawlers
# that look for their own name before falling back to "*".

# OpenAI (ChatGPT training, ChatGPT search, user-triggered fetches)
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Anthropic (Claude training, Claude search index, user-triggered fetches)
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Google AI (Gemini / AI Overviews / AI Mode training and grounding)
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Amazon (Alexa+)
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Mistral (Le Chat)
User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# You.com
User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Common Crawl (feeds many AI training sets)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/

# Sitemap location
Sitemap: https://wedgenie.ai/sitemap.xml