{ "name": "Eight ready-made robots.txt files", "description": "Each file names every relevant crawler explicitly, so a later change is a one-line diff and the intent stays readable. Pick the trade-off, curl the file, append it to your robots.txt.", "url": "https://www.pathwren.workers.dev/policy", "canonical": "https://www.pathwren.workers.dev/policy/index.html", "formats": { "html": "https://www.pathwren.workers.dev/policy/index.html", "json": "https://www.pathwren.workers.dev/policy/index.json", "markdown": "https://www.pathwren.workers.dev/policy/index.md" }, "curl": "curl -s https://www.pathwren.workers.dev/robots/block-ai-training.txt >> robots.txt", "count": 8, "entries": [ { "slug": "allow-all", "title": "Allow everything, explicitly", "summary": "Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold.", "crawlers_named": 150, "robots_txt": "https://www.pathwren.workers.dev/robots/allow-all.txt", "html": "https://www.pathwren.workers.dev/policy/allow-all.html", "json": "https://www.pathwren.workers.dev/policy/allow-all.json" }, { "slug": "block-ai-training", "title": "Block AI training, keep AI search", "summary": "Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers.", "crawlers_named": 27, "robots_txt": "https://www.pathwren.workers.dev/robots/block-ai-training.txt", "html": "https://www.pathwren.workers.dev/policy/block-ai-training.html", "json": "https://www.pathwren.workers.dev/policy/block-ai-training.json" }, { "slug": "block-all-ai", "title": "Block every AI crawler", "summary": "Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed.", "crawlers_named": 77, "robots_txt": "https://www.pathwren.workers.dev/robots/block-all-ai.txt", "html": "https://www.pathwren.workers.dev/policy/block-all-ai.html", "json": "https://www.pathwren.workers.dev/policy/block-all-ai.json" }, { "slug": "block-datasets", "title": "Block corpus and dataset builders", "summary": "Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot.", "crawlers_named": 17, "robots_txt": "https://www.pathwren.workers.dev/robots/block-datasets.txt", "html": "https://www.pathwren.workers.dev/policy/block-datasets.html", "json": "https://www.pathwren.workers.dev/policy/block-datasets.json" }, { "slug": "allow-ai-search-only", "title": "Allow AI search and user fetches, block the rest", "summary": "Be findable and citable in assistants without contributing to training corpora.", "crawlers_named": 65, "robots_txt": "https://www.pathwren.workers.dev/robots/allow-ai-search-only.txt", "html": "https://www.pathwren.workers.dev/policy/allow-ai-search-only.html", "json": "https://www.pathwren.workers.dev/policy/allow-ai-search-only.json" }, { "slug": "block-seo-tools", "title": "Block SEO and backlink crawlers", "summary": "Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic.", "crawlers_named": 17, "robots_txt": "https://www.pathwren.workers.dev/robots/block-seo-tools.txt", "html": "https://www.pathwren.workers.dev/policy/block-seo-tools.html", "json": "https://www.pathwren.workers.dev/policy/block-seo-tools.json" }, { "slug": "block-disputed", "title": "Block the crawlers with disputed robots compliance", "summary": "The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead.", "crawlers_named": 18, "robots_txt": "https://www.pathwren.workers.dev/robots/block-disputed.txt", "html": "https://www.pathwren.workers.dev/policy/block-disputed.html", "json": "https://www.pathwren.workers.dev/policy/block-disputed.json" }, { "slug": "maximum-ai-visibility", "title": "Maximum AI visibility", "summary": "Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines.", "crawlers_named": 133, "robots_txt": "https://www.pathwren.workers.dev/robots/maximum-ai-visibility.txt", "html": "https://www.pathwren.workers.dev/policy/maximum-ai-visibility.html", "json": "https://www.pathwren.workers.dev/policy/maximum-ai-visibility.json" } ], "generated_at": "2026-09-03T06:29:54+00:00", "license": { "data": "CC0-1.0", "url": "https://creativecommons.org/publicdomain/zero/1.0/" }, "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.", "links": [ { "rel": "self", "href": "https://www.pathwren.workers.dev/policy/index.json", "type": "application/json" }, { "rel": "changes", "href": "https://www.pathwren.workers.dev/changes.json?since=111", "type": "application/json", "title": "What changed since your cursor — poll this instead of re-downloading this document", "cursor_param": "since", "head_cursor": 111, "min_poll_seconds": 21600, "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag." }, { "rel": "related", "href": "https://www.pathwren.workers.dev/policy/index.md", "type": "text/markdown" }, { "rel": "related", "href": "https://www.pathwren.workers.dev/documents.json", "type": "application/json", "title": "Every document here with its ETag and last-modified date" }, { "rel": "service-desc", "href": "https://www.pathwren.workers.dev/openapi.json", "type": "application/json", "title": "Every read endpoint, described formally" } ] }