# AI Crawler Index — policy: block-seo-tools
# Block SEO and backlink crawlers
# Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.
# Generated 2026-09-03 from https://www.pathwren.workers.dev/policy/block-seo-tools.html
# 17 crawlers named. Paste into robots.txt at your document root.

User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-ESI
Disallow: /

User-agent: SemrushBot-FT
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: *
Allow: /

Sitemap: https://www.pathwren.workers.dev/sitemap.xml