# Foldspace — foldspace.ai
# Welcome AI search/answer crawlers: we want ChatGPT, Claude, and Perplexity to
# find, read, and cite these pages.
#
# Content Signals (contentsignals.org) express how this content may be used. As a
# condition of accessing this website, you agree to abide by the following signals:
#   (a) content-signal = yes  -> you may collect content for that purpose.
#   (b) content-signal = no   -> you may not collect content for that purpose.
#   (c) signal absent         -> no preference is expressed.
#
#   search    building a search index and returning links/short excerpts.
#   ai-input  supplying content to an AI model at query time (RAG, grounding,
#             answer engines) - how ChatGPT/Claude/Perplexity cite these pages.
#   ai-train  training or fine-tuning a model on this content.
#
# search=yes and ai-input=yes: being cited in AI answers is the whole point.
# ai-train=no: positioning, messaging, and customer proof are competitive assets.
# We want models to *retrieve* them live, not absorb them into weights. (The
# developer docs at docs.foldspace.ai make the opposite call on ai-train, because
# SDK recall genuinely benefits from training.)
#
# NOTE: robots.txt groups do NOT inherit. A crawler obeys only its single most
# specific matching group, so a Content-Signal on `User-agent: *` is invisible to
# any bot that has its own group below. Every group therefore repeats the line.
# If you add a group, add its Content-Signal line too.
#
# DRAFTS: draft share links are deliberately NOT disallowed here. A crawler
# blocked by robots.txt never fetches the page, so it never sees the page's
# noindex and can still index the bare URL if it finds a link somewhere; a
# Disallow line would also announce where the previews live. Drafts are kept out
# by "noindex, nofollow, noarchive" on every response (robots meta and
# X-Robots-Tag) and by never being listed or linked: no sitemap, llms.txt, RSS or
# .md twin. `npm run verify:drafts` fails if a Disallow covering them is added.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# AI answer/search engines and on-demand fetchers, allowed explicitly so there is
# no ambiguity about whether they may read the site.

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Agent-optimized content surfaces (plain Markdown, no site chrome):
#   /llms.txt      index of the key pages
#   <any-page>.md  a Markdown twin of that page (also served by content
#                  negotiation - send `Accept: text/markdown` to any page URL)
Sitemap: https://foldspace.ai/sitemap-index.xml