{
 "slug": "semrushbot",
 "name": "SemrushBot",
 "operator": "Semrush",
 "operator_slug": "semrush",
 "category": "seo",
 "category_label": "SEO and backlink crawlers",
 "robots_token": "SemrushBot",
 "user_agent_substring": "SemrushBot",
 "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
 "respects_robots_txt": "documented",
 "respects_robots_txt_label": "obeys robots.txt (documented)",
 "verification_method": "reverse-dns",
 "verification_label": "reverse DNS",
 "published_ip_ranges_url": null,
 "ip_ranges_endpoint": null,
 "ipv4_prefix_count": 0,
 "ipv6_prefix_count": 0,
 "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.",
 "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.",
 "operator_docs": "https://www.semrush.com/bot/",
 "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
 "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
 "last_reviewed": "2026-09-03",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/data/agents.json",
   "type": "application/json",
   "title": "Every crawler record in one file"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/crawler/semrushbot.md",
   "type": "text/markdown"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ]
}