{
 "slug": "block-datasets",
 "title": "Block corpus and dataset builders",
 "summary": "Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.",
 "detail": "These are the highest-leverage blocks per line, because one crawl becomes many downstream training runs. It is also the block with the longest delay before it has any effect, and no effect at all on archives already published.",
 "crawler_count": 17,
 "crawlers": [
  "ai2bot",
  "ai2bot-dolma",
  "aihitbot",
  "awariorssbot",
  "awariosmartbot",
  "ccbot",
  "diffbot",
  "echoboxbot",
  "imagesiftbot",
  "img2dataset",
  "laiondownloader",
  "omgili",
  "omgilibot",
  "panscient",
  "thinkbot",
  "velenpublicwebcrawler",
  "yak"
 ],
 "robots_txt_url": "https://www.pathwren.workers.dev/robots/block-datasets.txt",
 "robots_txt": "# AI Crawler Index — policy: block-datasets\n# Block corpus and dataset builders\n# Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.\n# Generated 2026-09-03 from https://www.pathwren.workers.dev/policy/block-datasets.html\n# 17 crawlers named. Paste into robots.txt at your document root.\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: AwarioRssBot\nDisallow: /\n\nUser-agent: AwarioSmartBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: LAIONDownloader   # operator states robots.txt does not apply; enforce at the edge\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: omgilibot\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: Thinkbot   # compliance disputed; enforce at the edge\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.pathwren.workers.dev/sitemap.xml\n",
 "generated": "2026-09-03",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/policy/block-datasets.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/robots/block-datasets.txt",
   "type": "text/plain"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/data/agents.json",
   "type": "application/json",
   "title": "Every crawler record in one file"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ]
}