# Every crawler may read the whole site, AI crawlers included, except /api/,
# which is the extension's backend. /api/stats stays open: it is the public
# JSON that the landing page and llms.txt cite for the usage figures.
#
# The AI user-agents named below change nothing, since any crawler not named
# falls under "*" and gets the same rules. They are listed so a reader can see
# that letting AI crawlers in was a deliberate choice. They share one group so
# the rules are written once and cannot drift apart. Keep blank lines out of
# the group: older parsers, Python's among them, read one as its end.
#
# A plain-language summary for LLMs lives at https://tidywl.com/llms.txt

User-agent: *
# AI training crawlers
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
# AI search, and fetchers a user's question triggers at answer time
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Google-CloudVertexBot
# Allow comes first for parsers that stop at the first match; Google and Bing
# take the longest match, which gives the same answer.
Allow: /api/stats
Disallow: /api/

Sitemap: https://tidywl.com/sitemap.xml