# robots.txt for pylonseo.io
#
# Served as a real file by nginx (see web/nginx.conf: `location = /robots.txt`).
# Before this existed the path fell through to the SPA fallback and answered
# crawlers with ~236 KB of marketing HTML plus a noindex meta tag.
#
# The public marketing routes are prerendered at build time and are the only
# URLs worth indexing. Everything below is either the app itself (behind auth),
# the admin console, or the JSON API.
#
# This site also ships https://pylonseo.io/llms.txt and /llms-full.txt, which
# describe the same public content for language models and agents. Those files
# are not access control: they are linked from robots.txt, from the HTML head
# (rel="describedby") and from the site footer so both crawlers and humans can
# find them. See web/nginx.conf for how they are served.

User-agent: *
Allow: /

# Application surfaces (require a session; never useful in an index).
Disallow: /dashboard
Disallow: /login
Disallow: /signup
Disallow: /checkout
Disallow: /invite/
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password

# Admin console. The admin app is mounted under /admin in web/src/App.tsx, so
# one prefix rule covers every console page (users, orders, plans, SEO editor,
# staff, audit, ...). The bare rules below it are kept for historical safety in
# case any top-level admin route ever returns; the /seo$ anchor is required so
# the public marketing pages (/seo-audit, /seo-checker) stay crawlable.
Disallow: /admin
Disallow: /users
Disallow: /orders
Disallow: /subscriptions
Disallow: /plans
Disallow: /products
Disallow: /sites
Disallow: /support
Disallow: /webhooks
Disallow: /newsletter
Disallow: /ai-cost
Disallow: /seo$
Disallow: /staff
Disallow: /audit
# /assets is deliberately NOT disallowed: that is also the built CSS/JS bundle
# directory, and blocking it stops crawlers from rendering the site at all.

# JSON API - no HTML, nothing to index.
Disallow: /v1/

# ---- AI and answer-engine crawlers ----
#
# Pylon SEO exists to get a site found in classic search AND in AI answers, so
# these are explicitly allowed rather than left to silently inherit the
# wildcard rule. Every one of these operators publishes docs asking sites to
# state their preference explicitly, and an explicit group is auditable if this
# file is ever edited by a tool that does not understand what it is changing.
#
# These bots are NOT one switch. They split into three independent groups, and
# robots.txt controls each separately:
#
#   training   GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot,
#              Bytespider, meta-externalagent, cohere-ai
#   search     OAI-SearchBot, Claude-SearchBot, PerplexityBot, Bingbot
#   on-demand  ChatGPT-User, Claude-User, Claude-Web, Perplexity-User
#
# Blocking a training bot does not affect citation; blocking a search bot does.
# The audit in this very product scores a site down for blocking them
# (rule ai-robots-blocked), so the policy here is to allow all of them.
#
# IMPORTANT: a group must repeat the Disallow list from the wildcard group
# above. Naming these bots in their own group REPLACES the wildcard rules for
# them, so an AI group of "Allow: /" alone would hand AI crawlers the admin
# console, the dashboard and the JSON API. Keep the two lists in sync.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: cohere-ai
User-agent: MistralAI-User
User-agent: Amazonbot
User-agent: YouBot
User-agent: DuckAssistBot
Allow: /

Disallow: /dashboard
Disallow: /login
Disallow: /signup
Disallow: /checkout
Disallow: /invite/
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /admin
Disallow: /users
Disallow: /orders
Disallow: /subscriptions
Disallow: /plans
Disallow: /products
Disallow: /sites
Disallow: /support
Disallow: /webhooks
Disallow: /newsletter
Disallow: /ai-cost
Disallow: /seo$
Disallow: /staff
Disallow: /audit
Disallow: /v1/

Sitemap: https://pylonseo.io/sitemap.xml