{
 "name": "Bulk data",
 "description": "Every endpoint this index serves as a file: no key, no signup, no rate limit, CORS open, and every one of them is the same file the HTML pages are generated from.",
 "url": "https://www.pathwren.workers.dev/data",
 "canonical": "https://www.pathwren.workers.dev/data/index.html",
 "formats": {
  "html": "https://www.pathwren.workers.dev/data/index.html",
  "json": "https://www.pathwren.workers.dev/data/index.json",
  "markdown": "https://www.pathwren.workers.dev/data/index.md"
 },
 "curl": "curl -s https://www.pathwren.workers.dev/data/agents.json | jq '.crawlers[] | select(.category==\"ai-search\") | .robots_token'",
 "count": 9,
 "entries": [
  {
   "path": "/data/agents.csv",
   "url": "https://www.pathwren.workers.dev/data/agents.csv",
   "bytes": 24812,
   "content_type": "text/csv",
   "what": "The same table, flat"
  },
  {
   "path": "/data/agents.json",
   "url": "https://www.pathwren.workers.dev/data/agents.json",
   "bytes": 200594,
   "content_type": "application/json",
   "what": "Every crawler record in one file"
  },
  {
   "path": "/data/ip-sources.json",
   "url": "https://www.pathwren.workers.dev/data/ip-sources.json",
   "bytes": 8689,
   "content_type": "application/json",
   "what": "Which operators publish IP ranges, and where"
  },
  {
   "path": "/data/observed-clients.csv",
   "url": "https://www.pathwren.workers.dev/data/observed-clients.csv",
   "bytes": 59484,
   "content_type": "text/csv",
   "what": ""
  },
  {
   "path": "/data/observed-clients.json",
   "url": "https://www.pathwren.workers.dev/data/observed-clients.json",
   "bytes": 1607725,
   "content_type": "application/json",
   "what": "Every named client observed asking this host for something"
  },
  {
   "path": "/data/robots-tokens.txt",
   "url": "https://www.pathwren.workers.dev/data/robots-tokens.txt",
   "bytes": 2174,
   "content_type": "text/plain",
   "what": ""
  },
  {
   "path": "/data/ua-regex.json",
   "url": "https://www.pathwren.workers.dev/data/ua-regex.json",
   "bytes": 10294,
   "content_type": "application/json",
   "what": "Compiled user-agent regexes for a log classifier or a WAF"
  },
  {
   "path": "/data/ua-regex.txt",
   "url": "https://www.pathwren.workers.dev/data/ua-regex.txt",
   "bytes": 2170,
   "content_type": "text/plain",
   "what": ""
  },
  {
   "path": "/data/user-agents.txt",
   "url": "https://www.pathwren.workers.dev/data/user-agents.txt",
   "bytes": 2145,
   "content_type": "text/plain",
   "what": ""
  }
 ],
 "generated_at": "2026-09-03T06:29:54+00:00",
 "license": {
  "data": "CC0-1.0",
  "url": "https://creativecommons.org/publicdomain/zero/1.0/"
 },
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/data/index.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/data/index.md",
   "type": "text/markdown"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/documents.json",
   "type": "application/json",
   "title": "Every document here with its ETag and last-modified date"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ]
}