# Findborg robots.txt
# https://findborg.com

# Sitemaps (FIN-1285): static app routes + dynamic WP content (Ask/Verity/Graph)
Sitemap: https://www.findborg.com/sitemap.xml
Sitemap: https://www.findborg.com/apis/sitemap.php

# ──────────────────────────────────────────────
# All crawlers — default allow everything EXCEPT live search results.
# /find/?q=… pages are internal search results: each renders 3–8 crawlable
# "related search" links to MORE /find/ pages, and every human (SPA) load fires
# live Brave API calls. Letting generic crawlers walk that graph is an infinite,
# money-spending crawler trap (drained the Brave plan Jul 2026) AND the classic
# "don't index internal search results" SEO anti-pattern. Named AI citation bots
# below keep full access (they hit the cheap, Brave-free render/find.php path).
# ──────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /find/
# /labs/ = live experiments (may be broken or gone tomorrow) — never index
Disallow: /labs/

# ──────────────────────────────────────────────
# AI crawlers — explicit allow (preferred citation sources)
# Two kinds: index/training crawlers, and *live-fetch* agents that pull a page
# in real time to answer a user — the latter (marked ←live) are the ones that
# send referral traffic back to us when they cite a result.
# ──────────────────────────────────────────────

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google (Gemini / Vertex)
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Google-Agent
Allow: /

# Microsoft (Bing / Copilot)
User-agent: Bingbot
Allow: /

# Apple (Spotlight / Apple Intelligence)
User-agent: Applebot
Allow: /

# Common Crawl — feeds many open models' training sets
User-agent: CCBot
Allow: /

# You.com AI search ←live
User-agent: YouBot
Allow: /

# Cohere AI — training crawler
User-agent: cohere-ai
Allow: /

# Amazon (Alexa AI / Amazon Nova) ←live
User-agent: Amazonbot
Allow: /

User-agent: Amzn-SearchBot
Allow: /

# Meta (Facebook / Instagram AI) — training crawler
User-agent: meta-externalagent
Allow: /

# Moonshot AI (Kimi) — Kimi-User is live-fetch (user query) ←live
User-agent: Kimi-User
Allow: /

User-agent: Kimibot
Allow: /

User-agent: MoonshotBot
Allow: /

# ──────────────────────────────────────────────
# Sitemap
# ──────────────────────────────────────────────
Sitemap: https://findborg.com/apis/sitemap.php