# Formspring robots.txt
# Marketing + docs are fully indexable. AI crawlers explicitly allowed -
# we believe in transparent, citable content.
#
# Format: one explicit User-agent block per crawler, each with its own
# self-contained Disallow rules and a closing Allow. No shared/stacked
# User-agent lines (those tripped the PSI "robots.txt is not valid" flag).
# Blank lines separate every block so parsers never merge directives.

User-agent: *
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /billing
Disallow: /ai-insights
Disallow: /forms/
Disallow: /oauth/
Disallow: /passkeys/
Disallow: /user/
Disallow: /webhooks/
Disallow: /f/
Disallow: /impersonation/
Disallow: /stripe/
Disallow: /horizon/
Disallow: /telescope/
Disallow: /demo/thank-you
Allow: /

# AI search crawler - explicitly allowed for marketing & docs.
User-agent: GPTBot
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# OpenAI's search crawler (user-initiated browsing).
User-agent: OAI-SearchBot
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# Anthropic's user-initiated fetch crawler.
User-agent: ClaudeBot
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# Perplexity's crawler.
User-agent: PerplexityBot
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# Google's AI training opt-in token (separate from Googlebot).
User-agent: Google-Extended
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# Apple's AI training opt-in token (separate from Applebot).
User-agent: Applebot-Extended
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# Anthropic's training crawler (distinct from ClaudeBot, which serves
# user-initiated fetches). Same access policy as the other AI crawlers.
User-agent: anthropic-ai
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# Cohere's crawler. Same access policy as the other AI crawlers.
User-agent: cohere-ai
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

# CCBot (Common Crawl) is allowed for marketing & docs. Common Crawl is
# the seed corpus most LLM training datasets pull from, so allowing it
# maximizes long-term AI citability of our public content. To opt out of
# the training corpus instead, either delete this stanza or replace the
# Allow rule below with `Disallow: /`. Decision belongs to product.
User-agent: CCBot
Crawl-delay: 10
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /forms/
Disallow: /f/
Disallow: /webhooks/
Disallow: /demo/thank-you
Allow: /

Sitemap: https://formspring.io/sitemap.xml

# AI discovery (llms.txt standard)
# https://formspring.io/llms.txt
# https://formspring.io/llms-full.txt