{
 "name": "AI Crawler Index",
 "description": "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes — as JSON, CSV, robots.txt and regex.",
 "url": "https://www.pathwren.workers.dev",
 "generated_at": "2026-09-03T06:29:54+00:00",
 "crawlers": 150,
 "operators": 74,
 "policies": 8,
 "ip_sources": 15,
 "ipv4_prefixes": 1984,
 "ipv6_prefixes": 1062,
 "machine_entry_points": [
  "https://www.pathwren.workers.dev/documents.json",
  "https://www.pathwren.workers.dev/documents.txt",
  "https://www.pathwren.workers.dev/llms.txt",
  "https://www.pathwren.workers.dev/llms-full.txt",
  "https://www.pathwren.workers.dev/openapi.json",
  "https://www.pathwren.workers.dev/openapi.yaml",
  "https://www.pathwren.workers.dev/apis.json",
  "https://www.pathwren.workers.dev/data/agents.json",
  "https://www.pathwren.workers.dev/data/agents.csv",
  "https://www.pathwren.workers.dev/data/ua-regex.json",
  "https://www.pathwren.workers.dev/ip-ranges/all.json",
  "https://www.pathwren.workers.dev/ip-ranges/all.txt",
  "https://www.pathwren.workers.dev/status.json",
  "https://www.pathwren.workers.dev/feed.json",
  "https://www.pathwren.workers.dev/feed.xml",
  "https://www.pathwren.workers.dev/changes.json",
  "https://www.pathwren.workers.dev/sitemap.xml",
  "https://www.pathwren.workers.dev/.well-known/api-catalog",
  "https://www.pathwren.workers.dev/.well-known/apis.json",
  "https://www.pathwren.workers.dev/.well-known/openapi.json",
  "https://www.pathwren.workers.dev/.well-known/openapi.yaml",
  "https://www.pathwren.workers.dev/.well-known/api-onboarding",
  "https://www.pathwren.workers.dev/.well-known/ai-plugin.json",
  "https://www.pathwren.workers.dev/.well-known/mcp.json",
  "https://www.pathwren.workers.dev/register.json",
  "https://www.pathwren.workers.dev/reference.json",
  "https://www.pathwren.workers.dev/terms.json",
  "https://www.pathwren.workers.dev/privacy.json"
 ],
 "mcp_server": {
  "url": "https://www.pathwren.workers.dev/mcp",
  "transport": "streamable-http",
  "documentation": "https://www.pathwren.workers.dev/mcp.html",
  "name": "dev.workers.pathwren.www/ai-crawler-index"
 },
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/index.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/documents.json",
   "type": "application/json",
   "title": "Every document here with its ETag and last-modified date"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/data/agents.json",
   "type": "application/json",
   "title": "Every crawler record in one file"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/ip-ranges/all.json",
   "type": "application/json",
   "title": "Every operator-published prefix, unioned and grouped by source"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/llms.txt",
   "type": "text/plain",
   "title": "The whole map in one text file"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ],
 "document_ledger": {
  "json": "https://www.pathwren.workers.dev/documents.json",
  "tsv": "https://www.pathwren.workers.dev/documents.txt",
  "what": "every URL published here with a strong ETag and the date its bytes last changed — re-fetch only what moved"
 }
}