# robots.txt for NebulaKit
#
# Content Signals (https://contentsignals.org) declare how this content may
# be used after it is fetched. This deployment allows all three:
#
#   search   = yes  — index it and link to it from search results
#   ai-input = yes  — retrieve it to ground a live AI answer, with citation
#   ai-train = yes  — use it as AI training data
#
# Private surfaces (admin console, JSON APIs, auth flows, the reset and
# setup routes) stay Disallowed for every crawler, AI or otherwise.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/health
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /profile
Disallow: /reset
Disallow: /setup
Disallow: /media/

# Named AI crawlers. The policy matches the wildcard group above, but
# several operators only honour rules written against their own token.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Diffbot
User-agent: TimpiBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/health
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /profile
Disallow: /reset
Disallow: /setup
Disallow: /media/

Sitemap: https://nebulakit.starspace.group/sitemap.xml