{
 "name": "Config snippets — AI Crawler Index",
 "what": "Generated nginx, Caddy, Apache, Cloudflare Worker and Python for classifying and blocking AI crawlers, regenerated from the index so they never drift.",
 "url": "https://www.pathwren.workers.dev/snippet/index.json",
 "twin_of": "https://www.pathwren.workers.dev/snippet/index.html",
 "page": {
  "path": "/snippet/index.html",
  "url": "https://www.pathwren.workers.dev/snippet/index.html",
  "type": "text/html"
 },
 "generated_at": "2026-09-03T06:29:54+00:00",
 "generated_from": "the bytes of /snippet/index.html, by surfaces/ai-crawler-index/build.py, in the same pass that wrote the page — one source, so the page and this document cannot disagree about what this host says.",
 "license": {
  "document": "CC0-1.0",
  "url": "https://creativecommons.org/publicdomain/zero/1.0/"
 },
 "access": {
  "api_key": "none",
  "account": "none",
  "rate_limit": "none",
  "cors": "*",
  "auth": "none — every document here is a public GET"
 },
 "commands": [
  "curl -s https://www.pathwren.workers.dev/snippet/index.json",
  "curl -s https://www.pathwren.workers.dev/documents.json | jq -r '.documents[] | select(.path==\"/snippet/index.json\") | .etag'"
 ],
 "sections": [
  {
   "heading": "Config snippets",
   "text": [
    "Generated from the same records as everything else, so they cannot drift from the index."
   ],
   "commands": [],
   "tables": [
    {
     "headers": [
      "Snippet",
      "What it does"
     ],
     "rows": [
      [
       "nginx: classify and block by user-agent",
       "A map block that turns the user-agent into a category, so you can rate-limit training crawlers, allow AI search, and 403 the disputed ones without maintaining three separate if-chains."
      ],
      [
       "Caddy: block by user-agent",
       "A Caddy matcher plus respond directive. Caddy's header_regexp matcher is case-sensitive, so the pattern is written with an inline (?i) flag."
      ],
      [
       "Apache: .htaccess block",
       "BrowserMatchNoCase plus a Require expression. Works on shared hosting where you cannot edit the main config."
      ],
      [
       "Cloudflare Worker: classify at the edge",
       "Fetches this index's regex list at the edge and tags every request with a category header, so your origin can log AI traffic without shipping a bot list to every app."
      ],
      [
       "Python: classify a request",
       "Forty lines that load agents.json and return the category, operator and robots token for a user-agent string. The shape most log pipelines need."
      ]
     ]
    }
   ],
   "links": [
    "/snippet/nginx-map.html",
    "/snippet/caddy.html",
    "/snippet/apache-htaccess.html",
    "/snippet/cloudflare-worker.html",
    "/snippet/python-classify.html"
   ]
  }
 ],
 "machine_doors": [
  {
   "url": "https://www.pathwren.workers.dev/tools/?s=client-dossiers",
   "name": "6 keyless GET tools",
   "what": "The read-only MCP tools of this host as plain GET endpoints — no JSON-RPC, no key"
  },
  {
   "url": "https://www.pathwren.workers.dev/documents.json",
   "name": "documents.json",
   "what": "Every document here with its strong ETag and the date its bytes changed"
  },
  {
   "url": "https://www.pathwren.workers.dev/changes",
   "name": "changes",
   "what": "What moved since your cursor — poll this instead of re-downloading anything"
  },
  {
   "url": "https://www.pathwren.workers.dev/llms.txt",
   "name": "llms.txt",
   "what": "The whole map in one text file"
  },
  {
   "url": "https://www.pathwren.workers.dev/openapi.json",
   "name": "openapi.json",
   "what": "Every read endpoint, described formally"
  },
  {
   "url": "https://www.pathwren.workers.dev/.well-known/agent-card.json",
   "name": "agent card",
   "what": "A2A agent card"
  },
  {
   "url": "https://www.pathwren.workers.dev/mcp",
   "name": "mcp",
   "what": "MCP over JSON-RPC (POST)"
  },
  {
   "url": "https://www.pathwren.workers.dev/a2a",
   "name": "a2a",
   "what": "A2A (POST message/send)"
  }
 ],
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/snippet/index.json",
   "type": "application/json"
  },
  {
   "rel": "describes",
   "href": "https://www.pathwren.workers.dev/snippet/index.html",
   "type": "text/html",
   "title": "The page this document is the JSON twin of: Config snippets — AI Crawler Index"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/documents.json",
   "type": "application/json",
   "title": "Every document here with its ETag and last-modified date"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/data/agents.json",
   "type": "application/json",
   "title": "Every crawler record in one file"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  },
  {
   "rel": "describedby",
   "href": "https://www.pathwren.workers.dev/llms.txt",
   "type": "text/plain",
   "title": "The whole map in one text file"
  }
 ]
}