{
 "operator": "Ahrefs",
 "slug": "ahrefs",
 "docs": "https://ahrefs.com/robot",
 "crawler_count": 2,
 "crawlers": [
  {
   "slug": "ahrefsbot",
   "name": "AhrefsBot",
   "operator": "Ahrefs",
   "operator_slug": "ahrefs",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "AhrefsBot",
   "user_agent_substring": "AhrefsBot",
   "user_agent_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json",
   "ipv4_prefix_count": 81,
   "ipv6_prefix_count": 0,
   "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.",
   "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.",
   "operator_docs": "https://ahrefs.com/robot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "ahrefssiteaudit",
   "name": "AhrefsSiteAudit",
   "operator": "Ahrefs",
   "operator_slug": "ahrefs",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "AhrefsSiteAudit",
   "user_agent_substring": "AhrefsSiteAudit",
   "user_agent_example": "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json",
   "ipv4_prefix_count": 81,
   "ipv6_prefix_count": 0,
   "what_it_is": "Ahrefs' site-audit crawler, separate from AhrefsBot. Ahrefs documents that it obeys robots.txt by default, and that a verified site owner can ask for it to be allowed to ignore robots.txt on their own site so the audit can see disallowed sections.",
   "cost_of_blocking": "Site owners auditing your domain get an incomplete report. Blocking it saves bandwidth and costs you nothing in search.",
   "operator_docs": "https://ahrefs.com/robot",
   "html_url": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.json",
   "last_reviewed": "2026-09-03"
  }
 ],
 "ip_range_endpoints": [
  {
   "slug": "ahrefs-crawler",
   "label": "AhrefsBot and AhrefsSiteAudit",
   "url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json"
  }
 ]
}