# maddog.finance — crawl policy
#
# MADDOG exists to be read by AIs. Search engines, AI answer engines and AI
# training crawlers are all welcome on every page. This file is default-allow;
# the named stanzas below are declarative, so the policy survives even if a
# blanket Disallow ever gets prepended by mistake.
#
# NOTE: Cloudflare's managed robots.txt / AI Crawl Control will prepend its own
# block-AI directives ahead of this file if enabled on the zone. Keep that
# feature OFF in the Cloudflare dashboard so this policy is the one that serves.
#
# Token list verified against operator docs 2026-08. Grok (xAI) and DeepSeek
# publish no crawler token — nothing to allow; just don't bot-challenge them.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# --- AI search / answer engines, ad review + user-triggered fetchers ---

User-agent: OAI-SearchBot
Allow: /

User-agent: OAI-AdsBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: meta-webindexer
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: Amzn-SearchBot
Allow: /

User-agent: Amzn-User
Allow: /

User-agent: MistralAI-Index
Allow: /

User-agent: MistralAI-User
Allow: /

# --- Training crawlers — being in the training set is distribution ---

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: MistralAI-Training
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://maddog.finance/sitemap.xml