# Recomma — recomma.ai
# The marketing pages are open to every crawler, the AI ones by name; the
# application behind sign-in is not a page and is not listed.
# /login is not disallowed: every page links to it, and it carries a noindex
# that a crawler can only obey if it is allowed to read it.

User-agent: *
Allow: /
Disallow: /projects/
Disallow: /settings/
Disallow: /admin/
Disallow: /onboarding
Disallow: /oauth/

# Each assistant sends up to three crawlers, and a site can answer each one
# differently: one that trains, one that builds the search index answers
# are drawn from, and one that fetches a page because a user asked about it.
# All three are welcome here, named so no reader has to infer it.

# Training
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

# Search indexes
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# Fetched on a user's request
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

# Product tokens: no crawler of their own, a say in what the answers use
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

Sitemap: https://recomma.ai/sitemap.xml