User-agent: *
Allow: /

# Block raw source/internal paths
Disallow: /.agent/
Disallow: /DesignPrinciples/
Disallow: /Testing-temporary/

# Block tagged URL variants — keeps crawl budget on canonical URLs.
Disallow: /*?utm_*
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?msclkid=
Disallow: /*?mc_cid=
Disallow: /draft/
Disallow: /preview/

# AI / generative-search crawlers — explicit allow blocks. The default *
# above already permits these, but explicit User-agent declarations make
# the intent unambiguous and survive future policy shifts that could
# restrict default-* semantics for AI agents.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Bytespider
Allow: /

Sitemap: https://bayswatertransflow.com/sitemap-index.xml