{ "category": "seo", "label": "SEO and backlink crawlers", "description": "Commercial link-graph tooling. No user-facing effect either way, and usually a large share of your bot bandwidth.", "count": 17, "crawlers": [ { "slug": "ahrefsbot", "name": "AhrefsBot", "operator": "Ahrefs", "operator_slug": "ahrefs", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "AhrefsBot", "user_agent_substring": "AhrefsBot", "user_agent_example": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json", "ipv4_prefix_count": 81, "ipv6_prefix_count": 0, "what_it_is": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.", "cost_of_blocking": "No user-facing effect. Ahrefs honours Crawl-delay, so rate-limiting is usually better than blocking.", "operator_docs": "https://ahrefs.com/robot", "html_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/ahrefsbot.json", "last_reviewed": "2026-09-03" }, { "slug": "ahrefssiteaudit", "name": "AhrefsSiteAudit", "operator": "Ahrefs", "operator_slug": "ahrefs", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "AhrefsSiteAudit", "user_agent_substring": "AhrefsSiteAudit", "user_agent_example": "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/ahrefs-crawler.json", "ipv4_prefix_count": 81, "ipv6_prefix_count": 0, "what_it_is": "Ahrefs' site-audit crawler, separate from AhrefsBot. Ahrefs documents that it obeys robots.txt by default, and that a verified site owner can ask for it to be allowed to ignore robots.txt on their own site so the audit can see disallowed sections.", "cost_of_blocking": "Site owners auditing your domain get an incomplete report. Blocking it saves bandwidth and costs you nothing in search.", "operator_docs": "https://ahrefs.com/robot", "html_url": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.html", "json_url": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.json", "last_reviewed": "2026-09-03" }, { "slug": "barkrowler", "name": "Barkrowler", "operator": "Babbar", "operator_slug": "babbar", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "barkrowler", "user_agent_substring": "barkrowler", "user_agent_example": "Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Babbar's crawler, which builds the link graph behind their French-market SEO tooling.", "cost_of_blocking": "You leave Babbar's index. No effect on search or assistants.", "operator_docs": "https://babbar.tech/crawler", "html_url": "https://www.pathwren.workers.dev/crawler/barkrowler.html", "json_url": "https://www.pathwren.workers.dev/crawler/barkrowler.json", "last_reviewed": "2026-09-03" }, { "slug": "dataforseobot", "name": "DataForSeoBot", "operator": "DataForSEO", "operator_slug": "dataforseo", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "DataForSeoBot", "user_agent_substring": "DataForSeoBot", "user_agent_example": "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Builds the backlink and SERP datasets DataForSEO resells through its API, so one crawl reaches many downstream tools.", "cost_of_blocking": "You leave a dataset that a long tail of SEO products is built on. No user-facing effect.", "operator_docs": "https://dataforseo.com/dataforseo-bot", "html_url": "https://www.pathwren.workers.dev/crawler/dataforseobot.html", "json_url": "https://www.pathwren.workers.dev/crawler/dataforseobot.json", "last_reviewed": "2026-09-03" }, { "slug": "dotbot", "name": "DotBot", "operator": "Moz", "operator_slug": "moz", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "dotbot", "user_agent_substring": "dotbot", "user_agent_example": "Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Moz's crawler for Link Explorer. Moz documents that it respects robots.txt and that dotbot is the token to name.", "cost_of_blocking": "You leave Moz's link index, so Domain Authority and link reports about your site get thinner. Nothing a reader or an assistant sees changes.", "operator_docs": "https://moz.com/help/moz-procedures/crawlers/dotbot", "html_url": "https://www.pathwren.workers.dev/crawler/dotbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/dotbot.json", "last_reviewed": "2026-09-03" }, { "slug": "mj12bot", "name": "MJ12bot", "operator": "Majestic", "operator_slug": "majestic", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "MJ12bot", "user_agent_substring": "MJ12bot", "user_agent_example": "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Majestic's link-graph crawler, run as a distributed community project. Majestic states plainly that it cannot restrict the bot to a fixed set of addresses, and offers a pre-arranged ident string in the request headers instead.", "cost_of_blocking": "You leave the Majestic backlink index. No search or AI effect. It supports Crawl-delay, which is usually the better answer than a block.", "operator_docs": "https://mj12bot.com/", "html_url": "https://www.pathwren.workers.dev/crawler/mj12bot.html", "json_url": "https://www.pathwren.workers.dev/crawler/mj12bot.json", "last_reviewed": "2026-09-03" }, { "slug": "rogerbot", "name": "rogerbot", "operator": "Moz", "operator_slug": "moz", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "rogerbot", "user_agent_substring": "rogerbot", "user_agent_example": "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler@moz.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Moz's Campaign crawler, which audits a site its own owner registered. Moz states there is no IP range for it — identification is by user-agent only.", "cost_of_blocking": "Moz Pro site audits of your domain stop. If the domain is yours and you use Moz, blocking this breaks your own reports.", "operator_docs": "https://moz.com/help/moz-procedures/crawlers/rogerbot", "html_url": "https://www.pathwren.workers.dev/crawler/rogerbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/rogerbot.json", "last_reviewed": "2026-09-03" }, { "slug": "semrushbot", "name": "SemrushBot", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot", "user_agent_substring": "SemrushBot", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.", "cost_of_blocking": "Your competitors' Semrush reports get thinner, and so do yours. No user-facing effect.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot.json", "last_reviewed": "2026-09-03" }, { "slug": "semrushbot-ba", "name": "SemrushBot-BA", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-BA", "user_agent_substring": "SemrushBot-BA", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The Backlink Audit crawler. It re-checks links pointing at a customer's site, which means it lands on the sites doing the linking.", "cost_of_blocking": "None to you. It costs the site being audited a little accuracy in their backlink report.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.json", "last_reviewed": "2026-09-03" }, { "slug": "semrushbot-esi", "name": "SemrushBot-ESI", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-ESI", "user_agent_substring": "SemrushBot-ESI", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-ESI; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The crawler for Semrush Enterprise Site Intelligence, the enterprise tier's own site analysis.", "cost_of_blocking": "Enterprise customers lose analysis of your domain. Nothing user-facing.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.json", "last_reviewed": "2026-09-03" }, { "slug": "semrushbot-ft", "name": "SemrushBot-FT", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-FT", "user_agent_substring": "SemrushBot-FT", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-FT; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches full text for the Plagiarism Checker and similar text-comparison tools.", "cost_of_blocking": "Your text stops being compared against other people's submissions — which also means copies of your text are less likely to be caught.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.json", "last_reviewed": "2026-09-03" }, { "slug": "semrushbot-si", "name": "SemrushBot-SI", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-SI", "user_agent_substring": "SemrushBot-SI", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-SI; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches pages for the On Page SEO Checker and similar advisory tools.", "cost_of_blocking": "Nothing user-facing. Semrush customers lose on-page suggestions for pages on your domain.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-si.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-si.json", "last_reviewed": "2026-09-03" }, { "slug": "semrushbot-swa", "name": "SemrushBot-SWA", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SemrushBot-SWA", "user_agent_substring": "SemrushBot-SWA", "user_agent_example": "Mozilla/5.0 (compatible; SemrushBot-SWA; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Checks whether a URL is reachable, for the SEO Writing Assistant. One request per URL a writer references, not a crawl.", "cost_of_blocking": "Writers using Semrush's assistant see your links reported as unreachable.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.html", "json_url": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.json", "last_reviewed": "2026-09-03" }, { "slug": "seokicks", "name": "SEOkicks", "operator": "SEOkicks", "operator_slug": "seokicks", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SEOkicks", "user_agent_substring": "SEOkicks", "user_agent_example": "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "A German backlink index. Its documentation names SEOkicks as the user-agent to use in robots.txt.", "cost_of_blocking": "You leave a regional backlink index. Nothing else changes.", "operator_docs": "https://www.seokicks.de/robot.html", "html_url": "https://www.pathwren.workers.dev/crawler/seokicks.html", "json_url": "https://www.pathwren.workers.dev/crawler/seokicks.json", "last_reviewed": "2026-09-03" }, { "slug": "serpstatbot", "name": "serpstatbot", "operator": "Serpstat", "operator_slug": "serpstat", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "serpstatbot", "user_agent_substring": "serpstatbot", "user_agent_example": "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Serpstat's backlink crawler. It documents support for Crawl-delay up to 20 seconds, including a delay set on the * group.", "cost_of_blocking": "You leave Serpstat's link index. Try Crawl-delay first — they honour it, and a slow crawler is cheaper to keep than to fight.", "operator_docs": "https://serpstatbot.com/", "html_url": "https://www.pathwren.workers.dev/crawler/serpstatbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/serpstatbot.json", "last_reviewed": "2026-09-03" }, { "slug": "siteauditbot", "name": "SiteAuditBot", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SiteAuditBot", "user_agent_substring": "SiteAuditBot", "user_agent_example": "Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Semrush's Site Audit crawler: it walks a site a customer owns and reports technical SEO problems. Semrush names it as the token to block for that product.", "cost_of_blocking": "Site Audit reports on your own domain stop working. If a customer is auditing your site with your permission, blocking this breaks their tooling and nothing of yours.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/siteauditbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/siteauditbot.json", "last_reviewed": "2026-09-03" }, { "slug": "splitsignalbot", "name": "SplitSignalBot", "operator": "Semrush", "operator_slug": "semrush", "category": "seo", "category_label": "SEO and backlink crawlers", "robots_token": "SplitSignalBot", "user_agent_substring": "SplitSignalBot", "user_agent_example": "Mozilla/5.0 (compatible; SplitSignalBot; +http://www.semrush.com/bot.html)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Runs SEO A/B tests on a customer's own site with the SplitSignal tool.", "cost_of_blocking": "A site owner's own A/B testing stops. Only relevant on domains whose owner uses the product.", "operator_docs": "https://www.semrush.com/bot/", "html_url": "https://www.pathwren.workers.dev/crawler/splitsignalbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/splitsignalbot.json", "last_reviewed": "2026-09-03" } ] }