{
 "operator": "Common Crawl",
 "slug": "commoncrawl",
 "docs": "https://commoncrawl.org/faq",
 "crawler_count": 1,
 "crawlers": [
  {
   "slug": "ccbot",
   "name": "CCBot",
   "operator": "Common Crawl",
   "operator_slug": "commoncrawl",
   "category": "dataset",
   "category_label": "Corpus and dataset builders",
   "robots_token": "CCBot",
   "user_agent_substring": "CCBot",
   "user_agent_example": "CCBot/2.0 (https://commoncrawl.org/faq/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://index.commoncrawl.org/ccbot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/commoncrawl-ccbot.json",
   "ipv4_prefix_count": 4,
   "ipv6_prefix_count": 1,
   "what_it_is": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.",
   "cost_of_blocking": "Future Common Crawl snapshots exclude you, so downstream training sets lose you too — but only going forward. Existing snapshots are permanent and blocking today does not retract them.",
   "operator_docs": "https://commoncrawl.org/ccbot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ccbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ccbot.json",
   "last_reviewed": "2026-09-03"
  }
 ],
 "ip_range_endpoints": [
  {
   "slug": "commoncrawl-ccbot",
   "label": "CCBot",
   "url": "https://index.commoncrawl.org/ccbot.json",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/commoncrawl-ccbot.json"
  }
 ]
}