# Allow all crawlers
User-agent: *
Allow: /
# Disallow search pages (if any)
Disallow: /search
# LLM and AI Crawlers - ALLOW (important for AI training and answers)
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: CCBot
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: Diffbot
Allow: /
User-agent: Bytespider
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: YouBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: Meta-ExternalAgent
Allow: /
User-agent: cohere-ai
Allow: /
# Traditional search engines
#
# We publish three plain-text mirrors of the HTML docs, all for AI ingestion
# (see the AI crawler groups above, which keep full access):
#
# /docs-raw/**.txt one .txt per page
# /llms-full.txt every page concatenated
# /**.md Markdown twin of each page, one per canonical route,
# linked from llms.txt and from each page's
#
#
# Search engines must NOT crawl any of them: a .txt or .md file cannot carry a
# canonical or a meta-robots tag, and we cannot set X-Robots-Tag headers on
# GitHub Pages, so robots.txt is the only lever. Left crawlable they put a
# second full copy of all ~300 pages into the search index and compete with
# the canonical HTML.
#
# The .md twins were added after this block was first written and were left
# uncovered by mistake; sealmetrics.com has carried the same Disallow since
# its own twins shipped.
#
# This does NOT affect AI Overviews or AI Mode, which are served from the
# HTML pages Googlebot still crawls in full.
#
# Do not "simplify" these back to a bare Allow: /
User-agent: Googlebot
Disallow: /docs-raw/
Disallow: /llms-full.txt
Disallow: /*.md$
Disallow: /search
Allow: /
User-agent: Bingbot
Disallow: /docs-raw/
Disallow: /llms-full.txt
Disallow: /*.md$
Disallow: /search
Allow: /
# Sitemap
Sitemap: https://docs.sealmetrics.com/sitemap.xml
# LLM-friendly content (llmstxt.org convention)
# Open to AI crawlers, closed to Googlebot/Bingbot — see the note above.
# Documentation index for AI agents: https://docs.sealmetrics.com/llms.txt
# Full documentation content: https://docs.sealmetrics.com/llms-full.txt
# Plain-text mirror of every page: https://docs.sealmetrics.com/docs-raw/.txt
# Markdown twin of every page: https://docs.sealmetrics.com/.md