# AI Crawler Index — Caddy
# curl -s https://www.pathwren.workers.dev/snippet/caddy.txt
@ai_training header_regexp User-Agent "(?i)(GPTBot|ClaudeBot|anthropic\-ai|GoogleOther|Bytespider|TikTokSpider|meta\-externalagent|FacebookBot|cohere\-training\-data\-crawler|Webzio\-Extended|SemrushBot\-OCOB|GoogleOther\-Image|GoogleOther\-Video|YandexAdditional|YandexAdditionalBot|SBIntuitionsBot|ICC\-Crawler|Cotoyogi|ISSCyberRiskCrawler|Sidetrade\ indexer\ bot|QuillBot|Factset_spyderbot|Poseidon\ Research\ Crawler|Reflectionbot|Linguee\ Bot)"
@disputed    header_regexp User-Agent "(?i)(Bytespider|TikTokSpider|ISSCyberRiskCrawler|Thinkbot|Linguee\ Bot)"

handle @disputed {
	respond "403 - see /robots.txt" 403
}
handle @ai_training {
	header X-Crawler-Class "ai-training"
	# file_server / reverse_proxy as usual
}