# beamline.io: every public page is open to search engines, AI answer engines and agents.
# Content signals (contentsignals.org): search, AI answers and AI training are all welcome.
# Account, checkout and agent sign-in pages carry noindex instead of being hidden here.

# AI search: these fetch the pages AI answers cite
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Bravebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /i/
Disallow: /r/
Disallow: /mcp
Disallow: /paddle/
Disallow: /nowpayments/
Disallow: /sso-callback
Disallow: /oauth-consent
Disallow: /code/
Disallow: /pkg/

# AI training
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: cohere-ai
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /i/
Disallow: /r/
Disallow: /mcp
Disallow: /paddle/
Disallow: /nowpayments/
Disallow: /sso-callback
Disallow: /oauth-consent
Disallow: /code/
Disallow: /pkg/

# Fetches a person asked an assistant for
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Meta-ExternalFetcher
User-agent: Gemini-Deep-Research
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /i/
Disallow: /r/
Disallow: /mcp
Disallow: /paddle/
Disallow: /nowpayments/
Disallow: /sso-callback
Disallow: /oauth-consent
Disallow: /code/
Disallow: /pkg/

# Everyone else, search engines included
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /i/
Disallow: /r/
Disallow: /mcp
Disallow: /paddle/
Disallow: /nowpayments/
Disallow: /sso-callback
Disallow: /oauth-consent
Disallow: /code/
Disallow: /pkg/

# Support knowledge is an output of the original docs, never a source for itself.
User-agent: SupportStudio
Allow: /
Disallow: /api/
Disallow: /i/
Disallow: /r/
Disallow: /mcp
Disallow: /paddle/
Disallow: /nowpayments/
Disallow: /sso-callback
Disallow: /oauth-consent
Disallow: /code/
Disallow: /pkg/
Disallow: /knowledge

Sitemap: https://beamline.io/sitemap.xml
# Agents: https://beamline.io/llms.txt · https://beamline.io/.well-known/ai-catalog.json · https://beamline.io/.well-known/mcp/server-card.json