{
 "slug": "nginx-map",
 "title": "nginx: classify and block by user-agent",
 "language": "nginx",
 "suggested_filename": "nginx-map.conf",
 "description": "A map block that turns the user-agent into a category, so you can rate-limit training crawlers, allow AI search, and 403 the disputed ones without maintaining three separate if-chains.",
 "url": "https://www.pathwren.workers.dev/snippet/nginx-map.txt",
 "body": "# AI Crawler Index — nginx classification map\n# curl -s https://www.pathwren.workers.dev/snippet/nginx-map.txt\nmap $http_user_agent $ai_crawler {\n    default             \"\";\n    \"~*AdsBot\\-Google\"  \"tool\";\n    \"~*AdsBot\\-Google\\-Mobile\"  \"tool\";\n    \"~*AdsBot\\-Google\\-Mobile\\-Apps\"  \"tool\";\n    \"~*AhrefsBot\"  \"seo\";\n    \"~*AhrefsSiteAudit\"  \"seo\";\n    \"~*AI2Bot\"  \"dataset\";\n    \"~*Ai2Bot\\-Dolma\"  \"dataset\";\n    \"~*aiHitBot\"  \"dataset\";\n    \"~*AIWebIndex\"  \"ai-search\";\n    \"~*Amazonbot\"  \"ai-search\";\n    \"~*Andibot\"  \"ai-search\";\n    \"~*Anomura\"  \"ai-search\";\n    \"~*anthropic\\-ai\"  \"ai-training\";\n    \"~*APIs\\-Google\"  \"tool\";\n    \"~*Applebot\"  \"search\";\n    \"~*archive\\.org_bot\"  \"archive\";\n    \"~*atlassian\\-bot\"  \"ai-search\";\n    \"~*AwarioRssBot\"  \"dataset\";\n    \"~*AwarioSmartBot\"  \"dataset\";\n    \"~*Baiduspider\"  \"search\";\n    \"~*barkrowler\"  \"seo\";\n    \"~*bedrockbot\"  \"ai-search\";\n    \"~*bingbot\"  \"search\";\n    \"~*Bytespider\"  \"ai-training\";\n    \"~*CCBot\"  \"dataset\";\n    \"~*ChatGPT\\ Agent\"  \"user-fetch\";\n    \"~*ChatGPT\\-User\"  \"user-fetch\";\n    \"~*Claude\\-SearchBot\"  \"ai-search\";\n    \"~*Claude\\-User\"  \"user-fetch\";\n    \"~*Claude\\-Web\"  \"ai-search\";\n    \"~*ClaudeBot\"  \"ai-training\";\n    \"~*Cloudflare\\-AutoRAG\"  \"ai-search\";\n    \"~*cohere\\-ai\"  \"user-fetch\";\n    \"~*cohere\\-training\\-data\\-crawler\"  \"ai-training\";\n    \"~*Cotoyogi\"  \"ai-training\";\n    \"~*Crawl4AI\"  \"tool\";\n    \"~*Crawlspace\"  \"tool\";\n    \"~*DataForSeoBot\"  \"seo\";\n    \"~*Diffbot\"  \"dataset\";\n    \"~*dotbot\"  \"seo\";\n    \"~*DuckAssistBot\"  \"ai-search\";\n    \"~*DuckDuckBot\"  \"search\";\n    \"~*EchoboxBot\"  \"dataset\";\n    \"~*ExaSearchBot\"  \"ai-search\";\n    \"~*FacebookBot\"  \"ai-training\";\n    \"~*facebookexternalhit\"  \"preview\";\n    \"~*Factset_spyderbot\"  \"ai-training\";\n    \"~*FeedFetcher\\-Google\"  \"tool\";\n    \"~*FirecrawlAgent\"  \"tool\";\n    \"~*Google\\-Agent\"  \"user-fetch\";\n    \"~*Google\\-CloudVertexBot\"  \"ai-search\";\n    \"~*Google\\-CWS\"  \"tool\";\n    \"~*Google\\-GeminiNotebook\"  \"user-fetch\";\n    \"~*Google\\-InspectionTool\"  \"tool\";\n    \"~*Google\\-Pinpoint\"  \"user-fetch\";\n    \"~*Google\\-Read\\-Aloud\"  \"user-fetch\";\n    \"~*Google\\-Safety\"  \"tool\";\n    \"~*Google\\-Site\\-Verification\"  \"tool\";\n    \"~*Googlebot\"  \"search\";\n    \"~*Googlebot\\-Image\"  \"search\";\n    \"~*Googlebot\\-News\"  \"search\";\n    \"~*Googlebot\\-Video\"  \"search\";\n    \"~*GoogleMessages\"  \"preview\";\n    \"~*GoogleOther\"  \"ai-training\";\n    \"~*GoogleOther\\-Image\"  \"ai-training\";\n    \"~*GoogleOther\\-Video\"  \"ai-training\";\n    \"~*GoogleProducer\"  \"tool\";\n    \"~*GPTBot\"  \"ai-training\";\n    \"~*ia_archiver\"  \"archive\";\n    \"~*ICC\\-Crawler\"  \"ai-training\";\n    \"~*ImagesiftBot\"  \"dataset\";\n    \"~*img2dataset\"  \"dataset\";\n    \"~*ISSCyberRiskCrawler\"  \"ai-training\";\n    \"~*Kagibot\"  \"search\";\n    \"~*KlaviyoAIBot\"  \"ai-search\";\n    \"~*LAIONDownloader\"  \"dataset\";\n    \"~*Lightpanda\"  \"tool\";\n    \"~*Linguee\\ Bot\"  \"ai-training\";\n    \"~*Mediapartners\\-Google\"  \"tool\";\n    \"~*meta\\-externalagent\"  \"ai-training\";\n    \"~*meta\\-externalfetcher\"  \"user-fetch\";\n    \"~*Meta\\-WebIndexer\"  \"ai-search\";\n    \"~*MistralAI\\-User\"  \"user-fetch\";\n    \"~*MJ12bot\"  \"seo\";\n    \"~*MojeekBot\"  \"search\";\n    \"~*OAI\\-SearchBot\"  \"ai-search\";\n    \"~*omgili\"  \"dataset\";\n    \"~*omgilibot\"  \"dataset\";\n    \"~*panscient\\.com\"  \"dataset\";\n    \"~*Perplexity\\-User\"  \"user-fetch\";\n    \"~*PerplexityBot\"  \"ai-search\";\n    \"~*PetalBot\"  \"search\";\n    \"~*PhindBot\"  \"ai-search\";\n    \"~*Pinterestbot\"  \"search\";\n    \"~*Poseidon\\ Research\\ Crawler\"  \"ai-training\";\n    \"~*QualifiedBot\"  \"ai-search\";\n    \"~*QuillBot\"  \"ai-training\";\n    \"~*Qwantbot\"  \"search\";\n    \"~*Qwantbot\\-news\"  \"search\";\n    \"~*Reflectionbot\"  \"ai-training\";\n    \"~*rogerbot\"  \"seo\";\n    \"~*SBIntuitionsBot\"  \"ai-training\";\n    \"~*Scrapy\"  \"tool\";\n    \"~*Screaming\\ Frog\\ SEO\\ Spider\"  \"tool\";\n    \"~*SemrushBot\"  \"seo\";\n    \"~*SemrushBot\\-BA\"  \"seo\";\n    \"~*SemrushBot\\-ESI\"  \"seo\";\n    \"~*SemrushBot\\-FT\"  \"seo\";\n    \"~*SemrushBot\\-OCOB\"  \"ai-training\";\n    \"~*SemrushBot\\-SI\"  \"seo\";\n    \"~*SemrushBot\\-SWA\"  \"seo\";\n    \"~*SEOkicks\"  \"seo\";\n    \"~*serpstatbot\"  \"seo\";\n    \"~*SeznamBot\"  \"search\";\n    \"~*ShapBot\"  \"ai-search\";\n    \"~*Sidetrade\\ indexer\\ bot\"  \"ai-training\";\n    \"~*SiteAuditBot\"  \"seo\";\n    \"~*Slackbot\"  \"preview\";\n    \"~*Slackbot\\-LinkExpanding\"  \"preview\";\n    \"~*SplitSignalBot\"  \"seo\";\n    \"~*Storebot\\-Google\"  \"search\";\n    \"~*TerraCotta\"  \"ai-search\";\n    \"~*Thinkbot\"  \"dataset\";\n    \"~*TikTokSpider\"  \"ai-training\";\n    \"~*Timpibot\"  \"search\";\n    \"~*VelenPublicWebCrawler\"  \"dataset\";\n    \"~*Webzio\\-Extended\"  \"ai-training\";\n    \"~*wpbot\"  \"tool\";\n    \"~*YaK\"  \"dataset\";\n    \"~*YandexAdditional\"  \"ai-training\";\n    \"~*YandexAdditionalBot\"  \"ai-training\";\n    \"~*YandexBlogs\"  \"search\";\n    \"~*YandexBot\"  \"search\";\n    \"~*YandexCalendar\"  \"user-fetch\";\n    \"~*YandexComBot\"  \"search\";\n    \"~*YandexDirect\"  \"tool\";\n    \"~*YandexFavicons\"  \"search\";\n    \"~*YandexImages\"  \"search\";\n    \"~*YandexMarket\"  \"search\";\n    \"~*YandexMedia\"  \"search\";\n    \"~*YandexMetrika\"  \"tool\";\n    \"~*YandexMobileBot\"  \"search\";\n    \"~*YandexRenderResourcesBot\"  \"search\";\n    \"~*YandexScreenshotBot\"  \"tool\";\n    \"~*YandexVideo\"  \"search\";\n    \"~*YandexWebmaster\"  \"tool\";\n    \"~*Yeti\"  \"search\";\n    \"~*YouBot\"  \"ai-search\";\n}\n\n# Then, in a server block: log it, rate-limit it, or refuse it.\n#   access_log /var/log/nginx/ai.log combined if=$ai_crawler;\n#   if ($ai_crawler = \"disputed\") { return 403; }\n"
}