# robots.txt for lumind.io
#
# The NOINDEX meta (set in layout.tsx) remains the active guardrail until the
# orchestrator lifts it. This file instructs crawlers and lists the sitemap and
# LLM descriptors so they are ready the moment indexing is enabled.

User-agent: *
Disallow: /api/
Disallow: /account/
Disallow: /success/
Disallow: /cancel/

# Allow the major AI search and retrieval crawlers explicitly.
# Current tokens (2026): ClaudeBot and Claude-SearchBot are Anthropic's active
# crawlers; Claude-Web is a deprecated token kept only as a harmless compat layer.
# Google-Extended controls Gemini training inclusion; GoogleOther is Google's
# non-search crawler; OAI-SearchBot indexes content for ChatGPT Search.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Applebot
Allow: /

User-agent: Amazonbot
Allow: /

# Blocked: bulk training scrapers we do not want using the content.
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

# Sitemap
Sitemap: https://lumind.io/sitemap.xml

# LLM discovery (non-standard, cited in GEO guides).
LLM-Descriptor: https://lumind.io/llms.txt
LLM-Descriptor-Full: https://lumind.io/llms-full.txt