# prostack.uk - crawl policy
#
# Everything is open to every crawler. The named blocks below add nothing to the
# wildcard's permissions; they exist so the AI-crawler position is stated rather
# than implied, and so tightening it later is a one-line edit in the right place
# instead of a research exercise.
#
# To restrict a bot, put a path (or "/") after its Disallow. To restrict every
# bot, change the wildcard block's Disallow. Note the two categories differ:
#   - retrieval/answer bots fetch a page to answer a live question and can cite
#     prostack.uk back to the user (OAI-SearchBot, PerplexityBot, ClaudeBot)
#   - training/dataset crawlers collect corpora and send no traffic back
#     (GPTBot, CCBot, Google-Extended, Applebot-Extended)
# Blocking the second group costs no search or answer-engine visibility.

# Content Signals (https://contentsignals.org/) — machine-readable statement of
# the same policy: pages may be used for search, live AI answers (ai-input) and
# AI training (ai-train). Keep this in step with the blocks below if tightened.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow:

# Search engines
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

# AI answer/retrieval crawlers
User-agent: OAI-SearchBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: ClaudeBot
Disallow:

# AI training/dataset crawlers
User-agent: GPTBot
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: Applebot-Extended
Disallow:

User-agent: CCBot
Disallow:

Sitemap: https://prostack.uk/sitemap-index.xml

# Curated site summary for LLMs: https://prostack.uk/llms.txt