{
 "operator": "Semrush",
 "slug": "semrush",
 "docs": "https://www.semrush.com/bot/",
 "crawler_count": 9,
 "crawlers": [
  {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot",
   "user_agent_substring": "SemrushBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
   "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "semrushbot-ba",
   "name": "SemrushBot-BA",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot-BA",
   "user_agent_substring": "SemrushBot-BA",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The Backlink Audit crawler. It re-checks links pointing at a customer's site, which means it lands on the sites doing the linking.",
   "cost_of_blocking": "None to you. It costs the site being audited a little accuracy in their backlink report.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "semrushbot-esi",
   "name": "SemrushBot-ESI",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot-ESI",
   "user_agent_substring": "SemrushBot-ESI",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-ESI; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The crawler for Semrush Enterprise Site Intelligence, the enterprise tier's own site analysis.",
   "cost_of_blocking": "Enterprise customers lose analysis of your domain. Nothing user-facing.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "semrushbot-ft",
   "name": "SemrushBot-FT",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot-FT",
   "user_agent_substring": "SemrushBot-FT",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-FT; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches full text for the Plagiarism Checker and similar text-comparison tools.",
   "cost_of_blocking": "Your text stops being compared against other people's submissions — which also means copies of your text are less likely to be caught.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "semrushbot-ocob",
   "name": "SemrushBot-OCOB",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "SemrushBot-OCOB",
   "user_agent_substring": "SemrushBot-OCOB",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "reverse-dns",
   "verification_label": "reverse DNS",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.",
   "cost_of_blocking": "Exclusion from Semrush's AI corpus, with its SEO crawl unaffected.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "semrushbot-si",
   "name": "SemrushBot-SI",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot-SI",
   "user_agent_substring": "SemrushBot-SI",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-SI; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches pages for the On Page SEO Checker and similar advisory tools.",
   "cost_of_blocking": "Nothing user-facing. Semrush customers lose on-page suggestions for pages on your domain.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-si.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-si.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "semrushbot-swa",
   "name": "SemrushBot-SWA",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SemrushBot-SWA",
   "user_agent_substring": "SemrushBot-SWA",
   "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-SWA; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Checks whether a URL is reachable, for the SEO Writing Assistant. One request per URL a writer references, not a crawl.",
   "cost_of_blocking": "Writers using Semrush's assistant see your links reported as unreachable.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "siteauditbot",
   "name": "SiteAuditBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SiteAuditBot",
   "user_agent_substring": "SiteAuditBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Semrush's Site Audit crawler: it walks a site a customer owns and reports technical SEO problems. Semrush names it as the token to block for that product.",
   "cost_of_blocking": "Site Audit reports on your own domain stop working. If a customer is auditing your site with your permission, blocking this breaks their tooling and nothing of yours.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/siteauditbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/siteauditbot.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "splitsignalbot",
   "name": "SplitSignalBot",
   "operator": "Semrush",
   "operator_slug": "semrush",
   "category": "seo",
   "category_label": "SEO and backlink crawlers",
   "robots_token": "SplitSignalBot",
   "user_agent_substring": "SplitSignalBot",
   "user_agent_example": "Mozilla/5.0 (compatible; SplitSignalBot; +http://www.semrush.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Runs SEO A/B tests on a customer's own site with the SplitSignal tool.",
   "cost_of_blocking": "A site owner's own A/B testing stops. Only relevant on domains whose owner uses the product.",
   "operator_docs": "https://www.semrush.com/bot/",
   "html_url": "https://www.pathwren.workers.dev/crawler/splitsignalbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/splitsignalbot.json",
   "last_reviewed": "2026-09-03"
  }
 ],
 "ip_range_endpoints": []
}