{ "openapi": "3.1.0", "info": { "title": "AI Crawler Index", "version": "2026-09-03", "summary": "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes — as JSON, CSV, robots.txt and regex.", "description": "A read-only, static, keyless index of 150 web crawlers operated by 74 companies and projects: what each one is for, the exact robots.txt token and user-agent substring, whether the operator says it obeys robots.txt, how to verify it is genuine, and — the part nobody else publishes — what you lose by blocking it.\n\nEvery path below is a static file. There is no key, no rate limit and no state; `Access-Control-Allow-Origin: *` is set, so it is callable from a browser. Data is CC0.\n\nThe IP-range mirrors are refreshed from 15 operator-published endpoints; /status.json reports when each was last fetched and whether it changed.", "license": { "name": "CC0-1.0", "url": "https://creativecommons.org/publicdomain/zero/1.0/" }, "termsOfService": "https://www.pathwren.workers.dev/terms.html", "contact": { "url": "https://www.pathwren.workers.dev/about.html" } }, "servers": [ { "url": "https://www.pathwren.workers.dev" } ], "externalDocs": { "description": "The same endpoints as a page, each with its curl.", "url": "https://www.pathwren.workers.dev/reference" }, "tags": [ { "name": "crawlers", "description": "One record per crawler." }, { "name": "bulk", "description": "The whole dataset in several shapes." }, { "name": "ip-ranges", "description": "Operator-published prefixes, normalised." }, { "name": "robots", "description": "Ready-made robots.txt policy files." }, { "name": "status", "description": "Freshness of the upstream sources." }, { "name": "changes", "description": "What changed since your last read, with a cursor to send back. The cheap way to stay current without re-downloading anything." }, { "name": "tools", "description": "The read-only tools this host runs over MCP, as keyless GET endpoints. Same implementation, no JSON-RPC session required." } ], "paths": { "/data/agents.json": { "get": { "summary": "Every crawler record, plus categories and an endpoint map", "operationId": "listCrawlers", "tags": [ "bulk" ], "responses": { "200": { "description": "150 records", "content": { "application/json": { "schema": { "type": "object", "properties": { "count": { "type": "integer" }, "generated_at": { "type": "string", "format": "date-time" }, "crawlers": { "type": "array", "items": { "$ref": "#/components/schemas/Crawler" } } } }, "example": { "count": 150, "generated_at": "2026-09-03T06:29:54+00:00", "crawlers": [ { "slug": "oai-searchbot", "name": "OAI-SearchBot", "operator": "OpenAI", "operator_slug": "openai", "category": "ai-search", "category_label": "AI search crawlers", "robots_token": "OAI-SearchBot", "user_agent_substring": "OAI-SearchBot", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/searchbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-searchbot.json", "ipv4_prefix_count": 35, "ipv6_prefix_count": 0, "what_it_is": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.", "cost_of_blocking": "High. Blocking this removes you from ChatGPT search results and from the source links ChatGPT shows. This is the single most expensive block on this list for anyone who wants to be cited by an assistant.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/oai-searchbot.json", "last_reviewed": "2026-09-03" } ] } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/{slug}.json": { "get": { "summary": "One crawler record by slug", "description": "The same records as /data/agents.json, one file each, so a client can fetch exactly the one it needs.", "operationId": "getCrawler", "tags": [ "crawlers" ], "parameters": [ { "name": "slug", "in": "path", "required": true, "description": "Crawler slug, from /data/agents.json", "schema": { "type": "string", "enum": [ "gptbot", "oai-searchbot", "chatgpt-user", "claudebot", "claude-searchbot", "claude-user", "anthropic-ai", "claude-web", "google-extended", "googlebot", "googleother", "google-cloudvertexbot", "google-inspectiontool", "googlebot-image", "googlebot-news", "storebot-google", "bingbot", "applebot", "applebot-extended", "perplexitybot", "perplexity-user", "ccbot", "bytespider", "tiktokspider", "meta-externalagent", "meta-externalfetcher", "facebookexternalhit", "facebookbot", "amazonbot", "duckassistbot", "duckduckbot", "ai2bot", "ai2bot-dolma", "cohere-ai", "cohere-training-data-crawler", "mistralai-user", "youbot", "diffbot", "omgilibot", "omgili", "webzio-extended", "imagesiftbot", "timpibot", "semrushbot", "semrushbot-ocob", "ahrefsbot", "archive-org-bot", "ia-archiver", "yandexbot", "baiduspider", "seznambot", "yeti", "petalbot", "firecrawlagent", "scrapy", "img2dataset", "googlebot-video", "googleother-image", "googleother-video", "apis-google", "adsbot-google", "adsbot-google-mobile", "adsbot-google-mobile-apps", "mediapartners-google", "google-safety", "feedfetcher-google", "google-read-aloud", "google-site-verification", "google-cws", "google-pinpoint", "googleproducer", "googlemessages", "google-gemininotebook", "google-agent", "yandeximages", "yandexvideo", "yandexmedia", "yandexblogs", "yandexmarket", "yandexwebmaster", "yandexmobilebot", "yandexfavicons", "yandexcalendar", "yandexdirect", "yandexmetrika", "yandexrenderresourcesbot", "yandexscreenshotbot", "yandexadditional", "yandexadditionalbot", "yandexcombot", "siteauditbot", "semrushbot-ba", "semrushbot-si", "semrushbot-swa", "splitsignalbot", "semrushbot-ft", "semrushbot-esi", "ahrefssiteaudit", "mj12bot", "dotbot", "rogerbot", "dataforseobot", "serpstatbot", "barkrowler", "screaming-frog-seo-spider", "seokicks", "mojeekbot", "kagibot", "qwantbot", "qwantbot-news", "slackbot-linkexpanding", "slackbot", "pinterestbot", "bedrockbot", "cloudflare-autorag", "exasearchbot", "shapbot", "terracotta", "crawlspace", "panscient", "sbintuitionsbot", "icc-crawler", "cotoyogi", "isscyberriskcrawler", "sidetrade-indexer-bot", "yak", "atlassian-bot", "klaviyoaibot", "quillbot", "phindbot", "andibot", "anomura", "aiwebindex", "factset-spyderbot", "poseidon-research-crawler", "qualifiedbot", "reflectionbot", "thinkbot", "aihitbot", "linguee-bot", "lightpanda", "laiondownloader", "velenpublicwebcrawler", "awariosmartbot", "awariorssbot", "echoboxbot", "meta-webindexer", "chatgpt-agent", "wpbot", "crawl4ai" ] } } ], "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" }, "example": { "slug": "gptbot", "name": "GPTBot", "operator": "OpenAI", "operator_slug": "openai", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "GPTBot", "user_agent_substring": "GPTBot", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/gptbot.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-gptbot.json", "ipv4_prefix_count": 21, "ipv6_prefix_count": 0, "what_it_is": "OpenAI's bulk crawler. Pages it fetches may be used to train future OpenAI foundation models. It is not the bot that puts you in ChatGPT's search results, and blocking it does not remove you from them.", "cost_of_blocking": "Your content is excluded from training data for future OpenAI models. No effect on ChatGPT search visibility, on citations, or on links a user pastes into ChatGPT.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/gptbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/gptbot.json", "last_reviewed": "2026-09-03" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "404": { "description": "No such crawler" } } } }, "/data/agents.csv": { "get": { "summary": "The same table as CSV", "operationId": "listCrawlersCsv", "tags": [ "bulk" ], "responses": { "200": { "description": "CSV with a header row", "content": { "text/csv": { "schema": { "type": "string" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/data/ua-regex.json": { "get": { "summary": "Pre-escaped user-agent regexes, whole-list and per category", "operationId": "getUserAgentRegex", "tags": [ "bulk" ], "responses": { "200": { "description": "Regex alternations", "content": { "application/json": { "schema": { "type": "object", "properties": { "all": { "type": "string" }, "ai_only": { "type": "string" }, "by_category": { "type": "object", "additionalProperties": { "type": "string" } } } } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/data/ip-sources.json": { "get": { "summary": "Which operators publish verifiable IP ranges, and where", "operationId": "listIpSources", "tags": [ "ip-ranges" ], "responses": { "200": { "description": "Source list", "content": { "application/json": {} }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/ip-ranges/all.json": { "get": { "summary": "Union of every operator-published prefix, grouped by source", "operationId": "getAllIpRanges", "tags": [ "ip-ranges" ], "responses": { "200": { "description": "1984 IPv4 and 1062 IPv6 prefixes", "content": { "application/json": {} }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/ip-ranges/all.txt": { "get": { "summary": "The same prefixes, one CIDR per line", "tags": [ "ip-ranges" ], "operationId": "getAllIpRangesText", "responses": { "200": { "description": "Plain text, # comments", "content": { "text/plain": { "schema": { "type": "string" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/ip-ranges/{source}.json": { "get": { "summary": "One operator's published prefix list, normalised", "operationId": "getIpRangeSource", "tags": [ "ip-ranges" ], "parameters": [ { "name": "source", "in": "path", "required": true, "description": "Source slug, from /data/ip-sources.json", "schema": { "type": "string", "enum": [ "openai-gptbot", "openai-searchbot", "openai-chatgpt-user", "google-googlebot", "google-special", "google-user-triggered", "google-user-triggered-google", "bing-bingbot", "apple-applebot", "duckduckgo-duckduckbot", "perplexity-bot", "perplexity-user", "google-user-triggered-agents", "commoncrawl-ccbot", "ahrefs-crawler" ] } } ], "responses": { "200": { "description": "Prefixes with provenance", "content": { "application/json": {} }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "404": { "description": "No such source" } } } }, "/robots/{policy}.txt": { "get": { "summary": "A ready-made robots.txt policy file", "operationId": "getRobotsPolicy", "tags": [ "robots" ], "parameters": [ { "name": "policy", "in": "path", "required": true, "schema": { "type": "string", "enum": [ "allow-all", "block-ai-training", "block-all-ai", "block-datasets", "allow-ai-search-only", "block-seo-tools", "block-disputed", "maximum-ai-visibility" ] } } ], "responses": { "200": { "description": "robots.txt fragment, ready to append", "content": { "text/plain": { "schema": { "type": "string" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/policy/{policy}.json": { "get": { "summary": "A policy with its rationale and the crawlers it names", "operationId": "getPolicy", "tags": [ "robots" ], "parameters": [ { "name": "policy", "in": "path", "required": true, "schema": { "type": "string", "enum": [ "allow-all", "block-ai-training", "block-all-ai", "block-datasets", "allow-ai-search-only", "block-seo-tools", "block-disputed", "maximum-ai-visibility" ] } } ], "responses": { "200": { "description": "Policy record", "content": { "application/json": {} }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/documents.json": { "get": { "summary": "The document ledger: every URL here, its ETag and its last-modified", "description": "One request that tells you what to re-fetch. Every document this host publishes is listed with a strong ETag (sha-256 over the exact bytes served, first 32 hex characters) and the date those bytes last CHANGED. Compare against your copy, then GET only the paths that differ — with If-None-Match set to the etag from this file, so even a wrong guess costs a 304 and no body. HTML pages are listed with a null etag: live counters are rendered into them per request, so no validator for one could be honest.", "operationId": "getDocumentLedger", "tags": [ "status" ], "responses": { "200": { "description": "Every published document with its validators", "content": { "application/json": { "schema": { "type": "object", "properties": { "generated_at": { "type": "string", "format": "date-time" }, "counts": { "type": "object" }, "documents": { "type": "array", "items": { "type": "object", "properties": { "path": { "type": "string" }, "kind": { "type": "string" }, "type": { "type": "string" }, "bytes": { "type": "integer" }, "etag": { "type": "string" }, "last_modified": { "type": "string" } } } } } } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/documents.txt": { "get": { "summary": "The same ledger as TSV: path, kind, bytes, etag, last-modified", "description": "For the client that is a shell. Comment lines start with '#'; every other line is five tab-separated fields. A '-' means no validator is published for that document.", "operationId": "getDocumentLedgerText", "tags": [ "status" ], "responses": { "200": { "description": "Tab-separated document ledger", "content": { "text/plain": {} }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/status.json": { "get": { "summary": "Freshness and health of every upstream IP-range endpoint", "operationId": "getStatus", "tags": [ "status" ], "responses": { "200": { "description": "Per-source fetch state", "content": { "application/json": { "schema": { "type": "object", "properties": { "sources_ok": { "type": "integer" }, "sources_failed": { "type": "integer" }, "generated_at": { "type": "string", "format": "date-time" } } } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/feed.json": { "get": { "summary": "JSON Feed 1.1 of what changed", "operationId": "getFeed", "tags": [ "status" ], "responses": { "200": { "description": "JSON Feed", "content": { "application/feed+json": {} }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/changes.json": { "get": { "summary": "What changed since your cursor", "operationId": "getChanges", "tags": [ "changes" ], "description": "A since-cursor feed of every real change to this index: operator IP-range lists that gained or lost prefixes, upstreams that started or stopped answering, and crawler records added, edited or withdrawn.\n\nRead `cursor` from the response and send it back as `since` next time. The cursor is a monotonic integer that only advances when something actually changed, so an unchanged answer is proof of nothing new — not a coincidence of timing. An empty page is about 2.5 KB; sending the response's ETag back as `If-None-Match` makes it a 304 with no body at all.\n\nThe 15 upstream endpoints are re-fetched every six hours, so polling faster than that returns the same cursor. Nothing is rate limited; the request is simply not worth your budget.\n\n`stale_sources` is repeated in EVERY response, including empty ones: when an upstream fails, this host keeps serving its last good prefixes rather than an empty list, and a client that polls daily should not have to have been listening at the exact minute it broke to find out.", "parameters": [ { "name": "since", "in": "query", "required": false, "description": "A cursor from a previous response, or an ISO-8601 timestamp. Omit it to read from the oldest retained event. An unparseable value is a 400, never a silently ignored parameter.", "schema": { "type": "string", "examples": [ "0", "12", "2026-09-01T00:00:00Z" ] } }, { "name": "limit", "in": "query", "required": false, "description": "Events per page, 1–400. Default 100. Follow `next` when `has_more` is true.", "schema": { "type": "integer", "minimum": 1, "maximum": 400, "default": 100 } } ], "responses": { "200": { "description": "A page of changes, oldest first", "headers": { "ETag": { "description": "Strong validator over these exact bytes. Send it as If-None-Match.", "schema": { "type": "string" } }, "Last-Modified": { "description": "When the newest fact in this answer was observed.", "schema": { "type": "string" } }, "X-Cursor": { "description": "The cursor to send as `since` next time; same as body.cursor.", "schema": { "type": "string" } } }, "content": { "application/json": { "schema": { "type": "object", "required": [ "cursor", "head", "count", "changes" ], "properties": { "cursor": { "type": "integer", "description": "Send this back as `since`." }, "head": { "type": "integer", "description": "Newest cursor that exists." }, "up_to_date": { "type": "boolean" }, "count": { "type": "integer" }, "has_more": { "type": "boolean" }, "next": { "type": "string", "format": "uri" }, "cursor_expired": { "type": "boolean", "description": "Your cursor is older than the oldest retained event, so this page cannot be complete. Re-read the documents in `refetch` once and poll from `cursor`." }, "stale_sources": { "type": "array", "description": "Upstreams that did not answer on the last refresh. Their last good prefixes are still served.", "items": { "type": "object" } }, "changes": { "type": "array", "items": { "$ref": "#/components/schemas/Change" } } } } } } }, "304": { "description": "Nothing has changed since the ETag or Last-Modified you sent. No body." }, "400": { "description": "`since` was neither a cursor nor a timestamp. The body names the accepted forms." } } } }, "/crawler/gptbot.json": { "get": { "summary": "Record for GPTBot", "operationId": "crawler_gptbot", "tags": [ "crawlers" ], "description": "OpenAI's bulk crawler. Pages it fetches may be used to train future OpenAI foundation models. It is not the bot that puts you in ChatGPT's search results, and blocking it does not remove you from them.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/oai-searchbot.json": { "get": { "summary": "Record for OAI-SearchBot", "operationId": "crawler_oai_searchbot", "tags": [ "crawlers" ], "description": "Builds the index ChatGPT search answers from. Content it collects is used for retrieval and citation, not for model training.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/chatgpt-user.json": { "get": { "summary": "Record for ChatGPT-User", "operationId": "crawler_chatgpt_user", "tags": [ "crawlers" ], "description": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/claudebot.json": { "get": { "summary": "Record for ClaudeBot", "operationId": "crawler_claudebot", "tags": [ "crawlers" ], "description": "Anthropic's bulk crawler, gathering pages that may be used to train Claude models.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/claude-searchbot.json": { "get": { "summary": "Record for Claude-SearchBot", "operationId": "crawler_claude_searchbot", "tags": [ "crawlers" ], "description": "Indexes pages so Claude's web search can find and cite them. Separate token from the training crawler, so search visibility and training consent are independent decisions.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/claude-user.json": { "get": { "summary": "Record for Claude-User", "operationId": "crawler_claude_user", "tags": [ "crawlers" ], "description": "Fetches a page because a Claude user asked Claude to read it, at that moment.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/anthropic-ai.json": { "get": { "summary": "Record for anthropic-ai", "operationId": "crawler_anthropic_ai", "tags": [ "crawlers" ], "description": "A legacy robots.txt token from before Anthropic consolidated on ClaudeBot. It is still widely present in robots.txt files and costs nothing to keep, but it is a control token rather than a bot you will see in logs.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/claude-web.json": { "get": { "summary": "Record for Claude-Web", "operationId": "crawler_claude_web", "tags": [ "crawlers" ], "description": "An earlier Anthropic token for user-facing web access, superseded by Claude-User and Claude-SearchBot. Kept here because it appears in most published robots.txt templates.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-extended.json": { "get": { "summary": "Record for Google-Extended", "operationId": "crawler_google_extended", "tags": [ "crawlers" ], "description": "Not a crawler. A robots.txt token that tells Google whether pages Googlebot already fetched may be used to train and ground Gemini. You will never see it in an access log; disallowing it changes what Google does with content it fetched under a different name.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googlebot.json": { "get": { "summary": "Record for Googlebot", "operationId": "crawler_googlebot", "tags": [ "crawlers" ], "description": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googleother.json": { "get": { "summary": "Record for GoogleOther", "operationId": "crawler_googleother", "tags": [ "crawlers" ], "description": "A generic fetcher used by Google product teams for one-off crawls and research, including data collection that does not belong to Search.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-cloudvertexbot.json": { "get": { "summary": "Record for Google-CloudVertexBot", "operationId": "crawler_google_cloudvertexbot", "tags": [ "crawlers" ], "description": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-inspectiontool.json": { "get": { "summary": "Record for Google-InspectionTool", "operationId": "crawler_google_inspectiontool", "tags": [ "crawlers" ], "description": "The fetcher behind Search Console's URL Inspection and the Rich Results Test. It runs when a site owner clicks a button.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googlebot-image.json": { "get": { "summary": "Record for Googlebot-Image", "operationId": "crawler_googlebot_image", "tags": [ "crawlers" ], "description": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googlebot-news.json": { "get": { "summary": "Record for Googlebot-News", "operationId": "crawler_googlebot_news", "tags": [ "crawlers" ], "description": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/storebot-google.json": { "get": { "summary": "Record for Storebot-Google", "operationId": "crawler_storebot_google", "tags": [ "crawlers" ], "description": "Checks shopping and checkout flows for Google's shopping surfaces.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/bingbot.json": { "get": { "summary": "Record for bingbot", "operationId": "crawler_bingbot", "tags": [ "crawlers" ], "description": "Bing's only crawler, and therefore also the crawler behind Microsoft Copilot's grounding. Microsoft's documented way to keep search indexing while refusing generative reuse is the nocache / noarchive robots meta directive, not a separate user-agent.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/applebot.json": { "get": { "summary": "Record for Applebot", "operationId": "crawler_applebot", "tags": [ "crawlers" ], "description": "Powers Siri, Spotlight and Safari suggestions. Blocking it is a search decision, not an AI decision — the AI decision has its own token.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/applebot-extended.json": { "get": { "summary": "Record for Applebot-Extended", "operationId": "crawler_applebot_extended", "tags": [ "crawlers" ], "description": "Apple's counterpart to Google-Extended: a robots.txt token that withdraws consent for Apple Intelligence and Apple foundation-model training, without touching Applebot's search crawl.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/perplexitybot.json": { "get": { "summary": "Record for PerplexityBot", "operationId": "crawler_perplexitybot", "tags": [ "crawlers" ], "description": "Builds Perplexity's search index. Perplexity is citation-heavy by product design, so inclusion here converts to referral traffic more directly than most AI surfaces.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/perplexity-user.json": { "get": { "summary": "Record for Perplexity-User", "operationId": "crawler_perplexity_user", "tags": [ "crawlers" ], "description": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/ccbot.json": { "get": { "summary": "Record for CCBot", "operationId": "crawler_ccbot", "tags": [ "crawlers" ], "description": "Common Crawl's corpus builder. It trains nothing itself, but its archive is an input to most open and many closed LLM training sets, which makes it the highest-leverage single entry on this list.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/bytespider.json": { "get": { "summary": "Record for Bytespider", "operationId": "crawler_bytespider", "tags": [ "crawlers" ], "description": "ByteDance's crawler, associated with training data collection for Doubao and related models. Repeatedly reported by CDNs and site operators as the highest-volume AI crawler on the web and as inconsistent about robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/tiktokspider.json": { "get": { "summary": "Record for TikTokSpider", "operationId": "crawler_tiktokspider", "tags": [ "crawlers" ], "description": "A second ByteDance crawler identifying with TikTok, collecting page content for the same family of models.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/meta-externalagent.json": { "get": { "summary": "Record for meta-externalagent", "operationId": "crawler_meta_externalagent", "tags": [ "crawlers" ], "description": "Meta's AI crawler, gathering training data for Llama and Meta AI. It replaced the older FacebookBot name for this purpose.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/meta-externalfetcher.json": { "get": { "summary": "Record for meta-externalfetcher", "operationId": "crawler_meta_externalfetcher", "tags": [ "crawlers" ], "description": "Fetches a page when a Meta AI user asks about a specific link.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/facebookexternalhit.json": { "get": { "summary": "Record for facebookexternalhit", "operationId": "crawler_facebookexternalhit", "tags": [ "crawlers" ], "description": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/facebookbot.json": { "get": { "summary": "Record for FacebookBot", "operationId": "crawler_facebookbot", "tags": [ "crawlers" ], "description": "Meta's older speech- and language-corpus crawler, largely superseded by meta-externalagent but still listed as a valid robots token.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/amazonbot.json": { "get": { "summary": "Record for Amazonbot", "operationId": "crawler_amazonbot", "tags": [ "crawlers" ], "description": "Amazon's crawler, feeding Alexa's ability to answer questions from the web and Amazon's own search and assistant products.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/duckassistbot.json": { "get": { "summary": "Record for DuckAssistBot", "operationId": "crawler_duckassistbot", "tags": [ "crawlers" ], "description": "Fetches pages so DuckAssist can generate and cite answers inside DuckDuckGo.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/duckduckbot.json": { "get": { "summary": "Record for DuckDuckBot", "operationId": "crawler_duckduckbot", "tags": [ "crawlers" ], "description": "DuckDuckGo's own crawler. Note that the bulk of DuckDuckGo's web results come from Bing, so blocking bingbot removes you from DuckDuckGo whether or not you allow this one.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/ai2bot.json": { "get": { "summary": "Record for AI2Bot", "operationId": "crawler_ai2bot", "tags": [ "crawlers" ], "description": "The Allen Institute's crawler, gathering pages for open research corpora such as Dolma that underpin fully open models like OLMo.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/ai2bot-dolma.json": { "get": { "summary": "Record for Ai2Bot-Dolma", "operationId": "crawler_ai2bot_dolma", "tags": [ "crawlers" ], "description": "The variant of AI2's crawler named for the Dolma corpus specifically.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/cohere-ai.json": { "get": { "summary": "Record for cohere-ai", "operationId": "crawler_cohere_ai", "tags": [ "crawlers" ], "description": "Cohere's fetcher, used when its assistant products need a page.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/cohere-training-data-crawler.json": { "get": { "summary": "Record for cohere-training-data-crawler", "operationId": "crawler_cohere_training_data_crawler", "tags": [ "crawlers" ], "description": "Cohere's separately-named bulk crawler for model training data, split out so consent for training and consent for retrieval can differ.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/mistralai-user.json": { "get": { "summary": "Record for MistralAI-User", "operationId": "crawler_mistralai_user", "tags": [ "crawlers" ], "description": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/youbot.json": { "get": { "summary": "Record for YouBot", "operationId": "crawler_youbot", "tags": [ "crawlers" ], "description": "You.com's crawler, feeding its AI search product and its search API.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/diffbot.json": { "get": { "summary": "Record for Diffbot", "operationId": "crawler_diffbot", "tags": [ "crawlers" ], "description": "Extracts structured records from pages to build a commercial knowledge graph that is resold and used for retrieval and training.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/omgilibot.json": { "get": { "summary": "Record for omgilibot", "operationId": "crawler_omgilibot", "tags": [ "crawlers" ], "description": "Webz.io's crawler, collecting web and forum text sold as datasets, including to model builders.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/omgili.json": { "get": { "summary": "Record for omgili", "operationId": "crawler_omgili", "tags": [ "crawlers" ], "description": "The older robots token for the same Webz.io collection, still honoured and still worth listing.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/webzio-extended.json": { "get": { "summary": "Record for Webzio-Extended", "operationId": "crawler_webzio_extended", "tags": [ "crawlers" ], "description": "Webz.io's opt-out token specifically for AI training reuse, in the pattern Google and Apple established.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/imagesiftbot.json": { "get": { "summary": "Record for ImagesiftBot", "operationId": "crawler_imagesiftbot", "tags": [ "crawlers" ], "description": "Crawls images for Hive AI's reverse-image and dataset products. Image-heavy sites see this one long before they see the text crawlers.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/timpibot.json": { "get": { "summary": "Record for Timpibot", "operationId": "crawler_timpibot", "tags": [ "crawlers" ], "description": "A distributed crawler building an independent search index outside the Google/Bing duopoly.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot.json": { "get": { "summary": "Record for SemrushBot", "operationId": "crawler_semrushbot", "tags": [ "crawlers" ], "description": "Semrush's backlink and keyword crawler. It is not an AI crawler, but it is usually in the top three by volume on any site, and it is the cheapest block on this list.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot-ocob.json": { "get": { "summary": "Record for SemrushBot-OCOB", "operationId": "crawler_semrushbot_ocob", "tags": [ "crawlers" ], "description": "Semrush's separately-tokenised crawler for its AI content tooling, split out so SEO crawling and AI reuse can be answered differently.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/ahrefsbot.json": { "get": { "summary": "Record for AhrefsBot", "operationId": "crawler_ahrefsbot", "tags": [ "crawlers" ], "description": "Ahrefs' backlink crawler, and one of the largest non-search crawlers on the web by request volume.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/archive-org-bot.json": { "get": { "summary": "Record for archive.org_bot", "operationId": "crawler_archive_org_bot", "tags": [ "crawlers" ], "description": "The Wayback Machine's crawler. Preservation rather than AI, but it lands in the same 'is this bot welcome' decision and its output is a public corpus.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/ia-archiver.json": { "get": { "summary": "Record for ia_archiver", "operationId": "crawler_ia_archiver", "tags": [ "crawlers" ], "description": "The legacy Alexa/Internet Archive token, still present in most robots.txt files and still occasionally honoured.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexbot.json": { "get": { "summary": "Record for YandexBot", "operationId": "crawler_yandexbot", "tags": [ "crawlers" ], "description": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/baiduspider.json": { "get": { "summary": "Record for Baiduspider", "operationId": "crawler_baiduspider", "tags": [ "crawlers" ], "description": "Baidu's search crawler, and the ingest path for Baidu's Ernie-backed answers.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/seznambot.json": { "get": { "summary": "Record for SeznamBot", "operationId": "crawler_seznambot", "tags": [ "crawlers" ], "description": "Seznam's crawler — the dominant search engine in the Czech Republic and one of the few national engines with its own index.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yeti.json": { "get": { "summary": "Record for Yeti", "operationId": "crawler_yeti", "tags": [ "crawlers" ], "description": "Naver's crawler. Naver is South Korea's largest search portal and runs its own index and its own generative answers.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/petalbot.json": { "get": { "summary": "Record for PetalBot", "operationId": "crawler_petalbot", "tags": [ "crawlers" ], "description": "Huawei's crawler for Petal Search, shipped as the default search on Huawei devices.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/firecrawlagent.json": { "get": { "summary": "Record for FirecrawlAgent", "operationId": "crawler_firecrawlagent", "tags": [ "crawlers" ], "description": "A hosted scrape-to-markdown service that LLM applications call to read pages. The requester is whoever is building on it, not Firecrawl itself, so volume and intent vary wildly.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/scrapy.json": { "get": { "summary": "Record for Scrapy", "operationId": "crawler_scrapy", "tags": [ "crawlers" ], "description": "Not an operator: the default user-agent of the most common Python crawling framework. Anyone can be behind it. Modern Scrapy obeys robots.txt by default, which is why the default UA is still worth a rule.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/img2dataset.json": { "get": { "summary": "Record for img2dataset", "operationId": "crawler_img2dataset", "tags": [ "crawlers" ], "description": "The tool used to turn image-URL lists such as LAION's into downloaded training sets. It is run by whoever is building a dataset, not by a single operator.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googlebot-video.json": { "get": { "summary": "Record for Googlebot-Video", "operationId": "crawler_googlebot_video", "tags": [ "crawlers" ], "description": "The video half of Googlebot. It crawls video files and the pages around them for Google Video search, and it is matched by a robots.txt group for Googlebot as well as by its own token.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googleother-image.json": { "get": { "summary": "Record for GoogleOther-Image", "operationId": "crawler_googleother_image", "tags": [ "crawlers" ], "description": "The image variant of GoogleOther: one-off fetches by Google product and research teams that are not Search. It also answers to a GoogleOther group in robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googleother-video.json": { "get": { "summary": "Record for GoogleOther-Video", "operationId": "crawler_googleother_video", "tags": [ "crawlers" ], "description": "The video variant of GoogleOther, used for internal Google fetches that do not belong to Search.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/apis-google.json": { "get": { "summary": "Record for APIs-Google", "operationId": "crawler_apis_google", "tags": [ "crawlers" ], "description": "Delivers push notifications for Google APIs to a webhook you registered. It is a special-case crawler: it ignores the robots.txt * group, because the fetch is a delivery to an address you asked it to deliver to.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/adsbot-google.json": { "get": { "summary": "Record for AdsBot-Google", "operationId": "crawler_adsbot_google", "tags": [ "crawlers" ], "description": "Checks the quality of desktop landing pages for Google Ads. Google documents that it ignores the robots.txt * group with the ad publisher's permission, and obeys a group named for its own token.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/adsbot-google-mobile.json": { "get": { "summary": "Record for AdsBot-Google-Mobile", "operationId": "crawler_adsbot_google_mobile", "tags": [ "crawlers" ], "description": "The mobile-web landing page checker for Google Ads. Same rules as AdsBot-Google: the * group does not apply to it, its own token does.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/adsbot-google-mobile-apps.json": { "get": { "summary": "Record for AdsBot-Google-Mobile-Apps", "operationId": "crawler_adsbot_google_mobile_apps", "tags": [ "crawlers" ], "description": "Checks Android app landing pages for Google Ads. It obeys a group named for its own token and, per Google, follows the AdsBot-Google rules otherwise.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/mediapartners-google.json": { "get": { "summary": "Record for Mediapartners-Google", "operationId": "crawler_mediapartners_google", "tags": [ "crawlers" ], "description": "The AdSense crawler. It reads a page so AdSense can choose relevant ads for it, and it is a special-case crawler that ignores the robots.txt * group.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-safety.json": { "get": { "summary": "Record for Google-Safety", "operationId": "crawler_google_safety", "tags": [ "crawlers" ], "description": "Google's abuse-investigation fetcher: malware review, phishing reports and similar. Google documents that it ignores robots.txt entirely, and a robots.txt rule for it does nothing.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/feedfetcher-google.json": { "get": { "summary": "Record for FeedFetcher-Google", "operationId": "crawler_feedfetcher_google", "tags": [ "crawlers" ], "description": "Crawls RSS and Atom feeds for Google News and WebSub. It is a user-triggered fetcher, and Google documents that those generally ignore robots.txt because a person asked for the fetch. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-read-aloud.json": { "get": { "summary": "Record for Google-Read-Aloud", "operationId": "crawler_google_read_aloud", "tags": [ "crawlers" ], "description": "Fetches a page so Google can read it out loud with text-to-speech, at the moment a user asks. Formerly google-speakr. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-site-verification.json": { "get": { "summary": "Record for Google-Site-Verification", "operationId": "crawler_google_site_verification", "tags": [ "crawlers" ], "description": "Fetches the token file or meta tag that proves you own a site, when you click verify in Search Console. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-cws.json": { "get": { "summary": "Record for Google-CWS", "operationId": "crawler_google_cws", "tags": [ "crawlers" ], "description": "The Chrome Web Store fetcher. It requests the URLs a developer put in the metadata of a Chrome extension or theme. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-pinpoint.json": { "get": { "summary": "Record for Google-Pinpoint", "operationId": "crawler_google_pinpoint", "tags": [ "crawlers" ], "description": "Fetches individual URLs that a Pinpoint user — usually a journalist or researcher — added as a source to their own document collection. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googleproducer.json": { "get": { "summary": "Record for GoogleProducer", "operationId": "crawler_googleproducer", "tags": [ "crawlers" ], "description": "Google Publisher Center: fetches the feeds a publisher explicitly supplied for Google News landing pages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/googlemessages.json": { "get": { "summary": "Record for GoogleMessages", "operationId": "crawler_googlemessages", "tags": [ "crawlers" ], "description": "Generates the link preview when somebody sends one of your URLs in Google Messages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-gemininotebook.json": { "get": { "summary": "Record for Google-GeminiNotebook", "operationId": "crawler_google_gemininotebook", "tags": [ "crawlers" ], "description": "Fetches a URL a Gemini Notebook (formerly NotebookLM) user added as a source to their notebook. The former agent string Google-NotebookLM is documented as supported until August 2026. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/google-agent.json": { "get": { "summary": "Record for Google-Agent", "operationId": "crawler_google_agent", "tags": [ "crawlers" ], "description": "Agents hosted on Google infrastructure navigating the web and taking actions on a user's request. Google names one prefix list for it — user-triggered-agents.json — and is separately experimenting with Web Bot Auth under the identity https://agent.bot.goog.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandeximages.json": { "get": { "summary": "Record for YandexImages", "operationId": "crawler_yandeximages", "tags": [ "crawlers" ], "description": "Indexes images for Yandex Images. Yandex's robot table marks it as taking the general robots.txt rules into account.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexvideo.json": { "get": { "summary": "Record for YandexVideo", "operationId": "crawler_yandexvideo", "tags": [ "crawlers" ], "description": "Indexes video for Yandex video search. Obeys the general robots.txt rules per Yandex's own table.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexmedia.json": { "get": { "summary": "Record for YandexMedia", "operationId": "crawler_yandexmedia", "tags": [ "crawlers" ], "description": "Indexes multimedia data for Yandex. Takes the general robots.txt rules into account.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexblogs.json": { "get": { "summary": "Record for YandexBlogs", "operationId": "crawler_yandexblogs", "tags": [ "crawlers" ], "description": "Yandex's blog-search robot; it indexes post comments as well as posts.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexmarket.json": { "get": { "summary": "Record for YandexMarket", "operationId": "crawler_yandexmarket", "tags": [ "crawlers" ], "description": "The robot behind Yandex Market, Yandex's shopping comparison service. Version 1.0 is documented as obeying the general rules; version 2.0 is documented as not.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexwebmaster.json": { "get": { "summary": "Record for YandexWebmaster", "operationId": "crawler_yandexwebmaster", "tags": [ "crawlers" ], "description": "The fetcher behind Yandex Webmaster, the console a site owner uses to inspect their own site.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexmobilebot.json": { "get": { "summary": "Record for YandexMobileBot", "operationId": "crawler_yandexmobilebot", "tags": [ "crawlers" ], "description": "Decides whether a page's layout is suitable for mobile devices. Yandex's table marks it as NOT taking the general robots.txt rules into account, so a * group does not stop it — a group named YandexMobileBot does.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexfavicons.json": { "get": { "summary": "Record for YandexFavicons", "operationId": "crawler_yandexfavicons", "tags": [ "crawlers" ], "description": "Downloads your favicon so Yandex can show it beside your result. Documented as not taking the general robots.txt rules into account.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexcalendar.json": { "get": { "summary": "Record for YandexCalendar", "operationId": "crawler_yandexcalendar", "tags": [ "crawlers" ], "description": "Downloads calendar files a user subscribed to. Yandex notes these files are often in directories that are disallowed for indexing, which is why the general rules are not applied.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexdirect.json": { "get": { "summary": "Record for YandexDirect", "operationId": "crawler_yandexdirect", "tags": [ "crawlers" ], "description": "Reads the content of Yandex Advertising Network partner pages to work out their topic so relevant ads can be matched. Documented as not taking the general robots.txt rules into account.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexmetrika.json": { "get": { "summary": "Record for YandexMetrika", "operationId": "crawler_yandexmetrika", "tags": [ "crawlers" ], "description": "Yandex Metrica's own fetcher. Two of its versions — the 2.0 yabs01 availability checker and the 4.0 CSS cache for Webvisor — are documented in Yandex's table as not using robots.txt at all.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexrenderresourcesbot.json": { "get": { "summary": "Record for YandexRenderResourcesBot", "operationId": "crawler_yandexrenderresourcesbot", "tags": [ "crawlers" ], "description": "Loads the CSS, JavaScript and images Yandex needs to render a page. Yandex documents the exact rule: it ignores robots.txt for a resource when the HTML page using it is allowed, and does not fetch the resource when that page is disallowed.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexscreenshotbot.json": { "get": { "summary": "Record for YandexScreenshotBot", "operationId": "crawler_yandexscreenshotbot", "tags": [ "crawlers" ], "description": "Takes a screenshot of a page. Documented as not taking the general robots.txt rules into account.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexadditional.json": { "get": { "summary": "Record for YandexAdditional", "operationId": "crawler_yandexadditional", "tags": [ "crawlers" ], "description": "The token that controls whether already-indexed pages may appear in Search with Yandex AI answers. Yandex's table says it makes no indexing requests of its own — it exists so a site can opt out of the generative answer without leaving the index.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexadditionalbot.json": { "get": { "summary": "Record for YandexAdditionalBot", "operationId": "crawler_yandexadditionalbot", "tags": [ "crawlers" ], "description": "The second token Yandex publishes for the same AI-answers opt-out. Both names appear in Yandex's own robot list, so a robots.txt that names only one of them is half a policy.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yandexcombot.json": { "get": { "summary": "Record for YandexComBot", "operationId": "crawler_yandexcombot", "tags": [ "crawlers" ], "description": "Indexes content for Yandex search in languages other than Russian. Yandex documents that it can index content when there is no explicit robot-specific restriction — a * group is not one.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/siteauditbot.json": { "get": { "summary": "Record for SiteAuditBot", "operationId": "crawler_siteauditbot", "tags": [ "crawlers" ], "description": "Semrush's Site Audit crawler: it walks a site a customer owns and reports technical SEO problems. Semrush names it as the token to block for that product.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot-ba.json": { "get": { "summary": "Record for SemrushBot-BA", "operationId": "crawler_semrushbot_ba", "tags": [ "crawlers" ], "description": "The Backlink Audit crawler. It re-checks links pointing at a customer's site, which means it lands on the sites doing the linking.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot-si.json": { "get": { "summary": "Record for SemrushBot-SI", "operationId": "crawler_semrushbot_si", "tags": [ "crawlers" ], "description": "Fetches pages for the On Page SEO Checker and similar advisory tools.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot-swa.json": { "get": { "summary": "Record for SemrushBot-SWA", "operationId": "crawler_semrushbot_swa", "tags": [ "crawlers" ], "description": "Checks whether a URL is reachable, for the SEO Writing Assistant. One request per URL a writer references, not a crawl.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/splitsignalbot.json": { "get": { "summary": "Record for SplitSignalBot", "operationId": "crawler_splitsignalbot", "tags": [ "crawlers" ], "description": "Runs SEO A/B tests on a customer's own site with the SplitSignal tool.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot-ft.json": { "get": { "summary": "Record for SemrushBot-FT", "operationId": "crawler_semrushbot_ft", "tags": [ "crawlers" ], "description": "Fetches full text for the Plagiarism Checker and similar text-comparison tools.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/semrushbot-esi.json": { "get": { "summary": "Record for SemrushBot-ESI", "operationId": "crawler_semrushbot_esi", "tags": [ "crawlers" ], "description": "The crawler for Semrush Enterprise Site Intelligence, the enterprise tier's own site analysis.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/ahrefssiteaudit.json": { "get": { "summary": "Record for AhrefsSiteAudit", "operationId": "crawler_ahrefssiteaudit", "tags": [ "crawlers" ], "description": "Ahrefs' site-audit crawler, separate from AhrefsBot. Ahrefs documents that it obeys robots.txt by default, and that a verified site owner can ask for it to be allowed to ignore robots.txt on their own site so the audit can see disallowed sections.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/mj12bot.json": { "get": { "summary": "Record for MJ12bot", "operationId": "crawler_mj12bot", "tags": [ "crawlers" ], "description": "Majestic's link-graph crawler, run as a distributed community project. Majestic states plainly that it cannot restrict the bot to a fixed set of addresses, and offers a pre-arranged ident string in the request headers instead.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/dotbot.json": { "get": { "summary": "Record for DotBot", "operationId": "crawler_dotbot", "tags": [ "crawlers" ], "description": "Moz's crawler for Link Explorer. Moz documents that it respects robots.txt and that dotbot is the token to name.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/rogerbot.json": { "get": { "summary": "Record for rogerbot", "operationId": "crawler_rogerbot", "tags": [ "crawlers" ], "description": "Moz's Campaign crawler, which audits a site its own owner registered. Moz states there is no IP range for it — identification is by user-agent only.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/dataforseobot.json": { "get": { "summary": "Record for DataForSeoBot", "operationId": "crawler_dataforseobot", "tags": [ "crawlers" ], "description": "Builds the backlink and SERP datasets DataForSEO resells through its API, so one crawl reaches many downstream tools.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/serpstatbot.json": { "get": { "summary": "Record for serpstatbot", "operationId": "crawler_serpstatbot", "tags": [ "crawlers" ], "description": "Serpstat's backlink crawler. It documents support for Crawl-delay up to 20 seconds, including a delay set on the * group.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/barkrowler.json": { "get": { "summary": "Record for Barkrowler", "operationId": "crawler_barkrowler", "tags": [ "crawlers" ], "description": "Babbar's crawler, which builds the link graph behind their French-market SEO tooling.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/screaming-frog-seo-spider.json": { "get": { "summary": "Record for Screaming Frog SEO Spider", "operationId": "crawler_screaming_frog_seo_spider", "tags": [ "crawlers" ], "description": "Not an operator: desktop crawling software that anybody can point at any site. The default user-agent identifies the tool, not who is running it, and the operator of the moment is whoever pressed start.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/seokicks.json": { "get": { "summary": "Record for SEOkicks", "operationId": "crawler_seokicks", "tags": [ "crawlers" ], "description": "A German backlink index. Its documentation names SEOkicks as the user-agent to use in robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/mojeekbot.json": { "get": { "summary": "Record for MojeekBot", "operationId": "crawler_mojeekbot", "tags": [ "crawlers" ], "description": "Mojeek's crawler. Mojeek runs one of the few genuinely independent web indexes — not a front end over Bing or Google — so it is one of the few blocks that removes you from an index nobody else can put you back into. Its documentation states it obeys the first record whose User-Agent contains MojeekBot, falling back to *.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/kagibot.json": { "get": { "summary": "Record for Kagibot", "operationId": "crawler_kagibot", "tags": [ "crawlers" ], "description": "The crawler for Kagi, a paid, ad-free search engine with its own index and its own assistant.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/qwantbot.json": { "get": { "summary": "Record for Qwantbot", "operationId": "crawler_qwantbot", "tags": [ "crawlers" ], "description": "Qwant's crawler. Qwant documents that the string Qwantbot always appears in its user-agents whatever the crawler version, which is what makes a substring match safe here.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/qwantbot-news.json": { "get": { "summary": "Record for Qwantbot-news", "operationId": "crawler_qwantbot_news", "tags": [ "crawlers" ], "description": "The news variant of Qwant's crawler, documented alongside the main one and carrying the same Qwantbot substring.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/slackbot-linkexpanding.json": { "get": { "summary": "Record for Slackbot-LinkExpanding", "operationId": "crawler_slackbot_linkexpanding", "tags": [ "crawlers" ], "description": "Fetches a page to build the unfurl card when somebody pastes your link into Slack. One paste, one fetch.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/slackbot.json": { "get": { "summary": "Record for Slackbot", "operationId": "crawler_slackbot", "tags": [ "crawlers" ], "description": "The other half of Slack's pair: the agent that reads robots.txt and handles Slack's non-unfurl fetches. Slack documents both strings on one page.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/pinterestbot.json": { "get": { "summary": "Record for Pinterestbot", "operationId": "crawler_pinterestbot", "tags": [ "crawlers" ], "description": "Pinterest's crawler. It indexes pages so people can find them on Pinterest and re-reads product pages to keep price and title on a Pin current. Pinterest states that content it crawls is not used to train their Canvas image generation model.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/bedrockbot.json": { "get": { "summary": "Record for bedrockbot", "operationId": "crawler_bedrockbot", "tags": [ "crawlers" ], "description": "The web crawler an AWS customer points at URLs they chose, to build a knowledge base for a Bedrock application. AWS documents that it respects robots.txt and that the user-agent carries a per-customer suffix, so you can allow or refuse one customer's crawl by naming bedrockbot-UUID.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/cloudflare-autorag.json": { "get": { "summary": "Record for Cloudflare-AutoRAG", "operationId": "crawler_cloudflare_autorag", "tags": [ "crawlers" ], "description": "The crawler behind Cloudflare's AI Search / AutoRAG, which indexes a website into a retrieval index for an application. Cloudflare's own documentation warns that a bot-blocking rule on your zone will also stop this crawler and tells you to allow-list it.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/exasearchbot.json": { "get": { "summary": "Record for ExaSearchBot", "operationId": "crawler_exasearchbot", "tags": [ "crawlers" ], "description": "Exa's crawler. It discovers and indexes public pages so they can be retrieved and cited through Exa's search API, which is one of the common retrieval backends behind agent frameworks.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/shapbot.json": { "get": { "summary": "Record for ShapBot", "operationId": "crawler_shapbot", "tags": [ "crawlers" ], "description": "Parallel's crawler. It collects and structures web content to power the search, extraction and deep-research APIs that Parallel sells to agent builders.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/terracotta.json": { "get": { "summary": "Record for TerraCotta", "operationId": "crawler_terracotta", "tags": [ "crawlers" ], "description": "Ceramic AI's crawler, which indexes public content for a web-scale search API aimed at LLMs and agents.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/crawlspace.json": { "get": { "summary": "Record for Crawlspace", "operationId": "crawler_crawlspace", "tags": [ "crawlers" ], "description": "A crawling platform: customers run their own crawls on it to feed agents, RAG pipelines and structured-data workflows. Like Firecrawl, the party behind any given request is the customer, not the platform.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/panscient.json": { "get": { "summary": "Record for Panscient", "operationId": "crawler_panscient", "tags": [ "crawlers" ], "description": "Compiles structured data about businesses and business professionals using machine learning. Panscient's FAQ states it obeys robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/sbintuitionsbot.json": { "get": { "summary": "Record for SBIntuitionsBot", "operationId": "crawler_sbintuitionsbot", "tags": [ "crawlers" ], "description": "SB Intuitions is SoftBank's Japanese LLM lab; this crawler gathers data used in that model development and in information analysis. The operator publishes a dedicated bot page.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/icc-crawler.json": { "get": { "summary": "Record for ICC-Crawler", "operationId": "crawler_icc_crawler", "tags": [ "crawlers" ], "description": "Operated by NICT, Japan's national information and communications research institute. The collected data supports AI research and, per the operator, is also provided to third parties including commercial companies.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/cotoyogi.json": { "get": { "summary": "Record for Cotoyogi", "operationId": "crawler_cotoyogi", "tags": [ "crawlers" ], "description": "A crawler run by ROIS-DS, a Japanese inter-university research organisation, collecting Japanese-language text for AI training. It publishes a crawler page in English and Japanese.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/isscyberriskcrawler.json": { "get": { "summary": "Record for ISSCyberRiskCrawler", "operationId": "crawler_isscyberriskcrawler", "tags": [ "crawlers" ], "description": "Crawls in order to train models that score a company's cyber risk. The ai.robots.txt dataset records the operator as not respecting robots.txt; ISS publishes no compliance statement of its own.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/sidetrade-indexer-bot.json": { "get": { "summary": "Record for Sidetrade indexer bot", "operationId": "crawler_sidetrade_indexer_bot", "tags": [ "crawlers" ], "description": "Sidetrade extracts web data for a range of uses including training its AI products for order-to-cash and customer-data work.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/yak.json": { "get": { "summary": "Record for YaK", "operationId": "crawler_yak", "tags": [ "crawlers" ], "description": "Meltwater's crawler, feeding the live data stream behind its media-monitoring and consumer-intelligence suite.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/atlassian-bot.json": { "get": { "summary": "Record for atlassian-bot", "operationId": "crawler_atlassian_bot", "tags": [ "crawlers" ], "description": "Indexes a website so it can be searched and cited by Rovo, Atlassian's generative assistant inside Jira and Confluence. Atlassian's documentation walks a customer through editing robots.txt for it, which is as close to a compliance statement as this list gets.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/klaviyoaibot.json": { "get": { "summary": "Record for KlaviyoAIBot", "operationId": "crawler_klaviyoaibot", "tags": [ "crawlers" ], "description": "Fetches pages from domains a Klaviyo customer has explicitly connected to their own account, to power Klaviyo's Kai customer agent. It is scoped to connected domains rather than the open web.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/quillbot.json": { "get": { "summary": "Record for QuillBot", "operationId": "crawler_quillbot", "tags": [ "crawlers" ], "description": "Operated by QuillBot as part of its writing, paraphrasing and AI-detection products. The dataset also records a second token, quillbot.com, for the same operator.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/phindbot.json": { "get": { "summary": "Record for PhindBot", "operationId": "crawler_phindbot", "tags": [ "crawlers" ], "description": "Phind is an answer engine for developers that combines live web search with its own models. This is the crawler behind those answers.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/andibot.json": { "get": { "summary": "Record for Andibot", "operationId": "crawler_andibot", "tags": [ "crawlers" ], "description": "The crawler for Andi, a small generative search assistant that summarises pages rather than listing them.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/anomura.json": { "get": { "summary": "Record for Anomura", "operationId": "crawler_anomura", "tags": [ "crawlers" ], "description": "Direqt's search crawler. It indexes the sites of Direqt's own publisher customers so their on-site chatbots can answer from them.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/aiwebindex.json": { "get": { "summary": "Record for AIWebIndex", "operationId": "crawler_aiwebindex", "tags": [ "crawlers" ], "description": "Builds an index of public pages and serves them to AI agents as extracted readable text, with attribution and a link back. Lyrenth publishes a crawler policy stating it does not train foundation models on what it collects and that it obeys robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/factset-spyderbot.json": { "get": { "summary": "Record for Factset_spyderbot", "operationId": "crawler_factset_spyderbot", "tags": [ "crawlers" ], "description": "FactSet's crawler, collecting data used in AI model training for its financial data and analytics products.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/poseidon-research-crawler.json": { "get": { "summary": "Record for Poseidon Research Crawler", "operationId": "crawler_poseidon_research_crawler", "tags": [ "crawlers" ], "description": "A crawler run by Poseidon Research, a lab working on interpretability research for AI systems.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/qualifiedbot.json": { "get": { "summary": "Record for QualifiedBot", "operationId": "crawler_qualifiedbot", "tags": [ "crawlers" ], "description": "Analyses a customer's website so Qualified's AI sales chatbots can answer questions about it in context.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/reflectionbot.json": { "get": { "summary": "Record for Reflectionbot", "operationId": "crawler_reflectionbot", "tags": [ "crawlers" ], "description": "An undocumented crawler whose user-agent links to Reflection AI, a company building AI models. The link in the user-agent is the only public statement of purpose that exists.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/thinkbot.json": { "get": { "summary": "Record for Thinkbot", "operationId": "crawler_thinkbot", "tags": [ "crawlers" ], "description": "Collects pages for analysis of how sites are adopting AI and automation. The ai.robots.txt dataset records the operator as not respecting robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/aihitbot.json": { "get": { "summary": "Record for aiHitBot", "operationId": "crawler_aihitbot", "tags": [ "crawlers" ], "description": "aiHit's automated collector, building a company dataset from public company websites.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/linguee-bot.json": { "get": { "summary": "Record for Linguee Bot", "operationId": "crawler_linguee_bot", "tags": [ "crawlers" ], "description": "Gathers bilingual text for Linguee's translation corpus and the machine translation trained on it. Recorded in the ai.robots.txt dataset as not respecting robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/lightpanda.json": { "get": { "summary": "Record for Lightpanda", "operationId": "crawler_lightpanda", "tags": [ "crawlers" ], "description": "A purpose-built headless browser for AI and automation — a runtime, not an operator. Whether robots.txt is honoured is left to whoever runs it, which is what its maintainers say themselves.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/laiondownloader.json": { "get": { "summary": "Record for LAIONDownloader", "operationId": "crawler_laiondownloader", "tags": [ "crawlers" ], "description": "LAION's downloader, used to materialise the image and text datasets the non-profit publishes for machine-learning research. LAION's own FAQ is the source for its robots.txt position.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/velenpublicwebcrawler.json": { "get": { "summary": "Record for VelenPublicWebCrawler", "operationId": "crawler_velenpublicwebcrawler", "tags": [ "crawlers" ], "description": "Hunter's crawler, written in Go, building business datasets and machine-learning models from public pages. Its page states it follows robots.txt and meta directives and never fetches more than one page every two seconds.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/awariosmartbot.json": { "get": { "summary": "Record for AwarioSmartBot", "operationId": "crawler_awariosmartbot", "tags": [ "crawlers" ], "description": "Awario's brand-monitoring crawler. It documents one request per three seconds, honours Crawl-delay, and states it does not use consecutive IP blocks so identification is by user-agent only.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/awariorssbot.json": { "get": { "summary": "Record for AwarioRssBot", "operationId": "crawler_awariorssbot", "tags": [ "crawlers" ], "description": "The feed-reading half of Awario's pair, documented on the same page and under the same crawl-rate policy.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/echoboxbot.json": { "get": { "summary": "Record for EchoboxBot", "operationId": "crawler_echoboxbot", "tags": [ "crawlers" ], "description": "Collects data supporting Echobox's AI-driven social and email distribution products, which publishers use to schedule and target their own content.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/meta-webindexer.json": { "get": { "summary": "Record for Meta-WebIndexer", "operationId": "crawler_meta_webindexer", "tags": [ "crawlers" ], "description": "Per Meta's crawler documentation, Meta-WebIndexer navigates the web to improve the quality of Meta AI's search results. It is a third Meta token alongside Meta-ExternalAgent and Meta-ExternalFetcher, and the newest of them.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/chatgpt-agent.json": { "get": { "summary": "Record for ChatGPT Agent", "operationId": "crawler_chatgpt_agent", "tags": [ "crawlers" ], "description": "ChatGPT's agent mode driving a real browser: it navigates and interacts with sites to finish a multi-step task a user gave it. OpenAI governs it with the ChatGPT-User token and the ChatGPT-User prefix list rather than a token of its own, so the robots rule and the address check are the same ones.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/wpbot.json": { "get": { "summary": "Record for wpbot", "operationId": "crawler_wpbot", "tags": [ "crawlers" ], "description": "Supports the AI Chatbot for WordPress plugin: it reads pages so the plugin can answer from a site's own content. The operator provides an opt-out through a form rather than through robots.txt.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/crawler/crawl4ai.json": { "get": { "summary": "Record for Crawl4AI", "operationId": "crawler_crawl4ai", "tags": [ "crawlers" ], "description": "An open-source LLM-oriented crawler and scraper library, run by whoever installs it. Like Scrapy, the default user-agent identifies the software and says nothing about who is behind the request.", "responses": { "200": { "description": "The crawler record", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Crawler" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } } } } }, "/tools/whoami": { "get": { "summary": "Who is calling? — the one endpoint that needs nothing from you", "operationId": "tool_whoami", "tags": [ "tools" ], "description": "Takes no arguments at all. It answers about the request you just made: the user-agent you sent, the address you came from, whether the crawler index recognises you, whether any operator publishes a prefix containing your address, the class this host's own instrument books you as, and whether we have seen you here before and what you fetched. Every fact comes from your own request headers or from a file this host already publishes; nothing is fetched and nothing about you is invented.\n\nNo account, no key, no handshake. This endpoint takes no parameters at all: a bare GET is the whole call and returns the real answer. Add ?spec=1 for the specification instead. It is the one call you can make here before you know anything about this host. This endpoint holds no copy of the logic: it calls the `whoami` MCP tool at https://www.pathwren.workers.dev/mcp in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/whoami.html · machine spec: https://www.pathwren.workers.dev/tools/whoami.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/whoami.html" }, "parameters": [ { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } } }, "/tools/example": { "get": { "summary": "A worked example, run for real — no arguments", "operationId": "tool_example", "tags": [ "tools" ], "description": "Takes no arguments at all. It runs this host's flagship tool (classify_user_agent) on a canned input from our own published log — the busiest client we have seen — and returns exactly the answer a real call returns, not a mock. Use it to see the shape of an answer before deciding what to send.\n\nNo account, no key, no handshake. This endpoint takes no parameters at all: a bare GET is the whole call and returns the real answer. Add ?spec=1 for the specification instead. It is the one call you can make here before you know anything about this host. This endpoint holds no copy of the logic: it calls the `example` MCP tool at https://www.pathwren.workers.dev/mcp in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/example.html · machine spec: https://www.pathwren.workers.dev/tools/example.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/example.html" }, "parameters": [ { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } } }, "/tools/verify-crawler": { "get": { "summary": "Does this address really belong to the crawler it claims to be?", "operationId": "tool_verify_crawler", "tags": [ "tools" ], "description": "Check one or more IP addresses against every crawler prefix the operators themselves publish (Google, OpenAI, Anthropic, Microsoft, Apple, Perplexity, Amazon and the rest, remirrored every six hours), and — when you name the crawler the client claimed to be — say whether the range that contains it belongs to that operator or to a different one.\n\nNo account, no key, no handshake. Called with no parameters at all it answers 200 with its own specification rather than 400, so walking /tools/ is free. This endpoint holds no copy of the logic: it calls the `verify_batch` MCP tool at https://www.pathwren.workers.dev/mcp/netcheck in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/verify-crawler.html · machine spec: https://www.pathwren.workers.dev/tools/verify-crawler.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/verify-crawler.html" }, "parameters": [ { "name": "ip", "in": "query", "required": true, "description": "IPv4 or IPv6 address. Repeat the parameter, or send a comma/space separated list. Up to 500 per call. Repeatable.", "schema": { "type": "array", "items": { "type": "string" } } }, { "name": "ua", "in": "query", "required": false, "description": "The crawler the client claimed to be — a product token like GPTBot or a whole User-Agent header. Applied to every address in the call.", "schema": { "type": "string" } }, { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } } }, "/tools/robots-lint": { "get": { "summary": "Lint a robots.txt", "operationId": "tool_robots_lint", "tags": [ "tools" ], "description": "Paste a robots.txt and get every fault that makes it do something other than what it looks like: misspelled directives, a whole User-Agent string where a product token belongs, rules before any User-agent line, duplicate groups, noindex (unsupported since 2019), relative Sitemap URLs, a byte-order mark. Each finding carries the line number and the fix.\n\nNo account, no key, no handshake. Called with no parameters at all it answers 200 with its own specification rather than 400, so walking /tools/ is free. This endpoint holds no copy of the logic: it calls the `lint_robots_txt` MCP tool at https://www.pathwren.workers.dev/mcp/robots in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/robots-lint.html · machine spec: https://www.pathwren.workers.dev/tools/robots-lint.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/robots-lint.html" }, "parameters": [ { "name": "robots_txt", "in": "query", "required": true, "description": "The contents of the robots.txt file, URL-encoded. Or POST the file as the raw request body, which is what a file larger than a URL wants.", "schema": { "type": "string" } }, { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } }, "post": { "summary": "Lint a robots.txt (file as the raw body)", "operationId": "tool_robots_lint_body", "tags": [ "tools" ], "description": "Identical to the GET, with the `robots_txt` argument taken from the raw request body instead of the query string — for files too large or too awkward to URL-encode. Still keyless.", "requestBody": { "required": true, "content": { "text/plain": { "schema": { "type": "string" } } } }, "responses": { "200": { "description": "The tool's answer.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "400": { "description": "The body was empty or unusable." } } } }, "/tools/robots-allowed": { "get": { "summary": "Would this crawler fetch this path?", "operationId": "tool_robots_allowed", "tags": [ "tools" ], "description": "Evaluate a pasted robots.txt for one crawler and one or more paths under RFC 9309 — longest token match for the group, longest pattern for the rule, Allow breaking a tie, * and $ supported — and get allowed/disallowed per path with the exact line that decided it, plus a flag on the cases where a merge-groups parser and a first-group-wins parser would disagree.\n\nNo account, no key, no handshake. Called with no parameters at all it answers 200 with its own specification rather than 400, so walking /tools/ is free. This endpoint holds no copy of the logic: it calls the `check_path_allowed` MCP tool at https://www.pathwren.workers.dev/mcp/robots in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/robots-allowed.html · machine spec: https://www.pathwren.workers.dev/tools/robots-allowed.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/robots-allowed.html" }, "parameters": [ { "name": "robots_txt", "in": "query", "required": true, "description": "The contents of the robots.txt file, URL-encoded, or the raw POST body.", "schema": { "type": "string" } }, { "name": "ua", "in": "query", "required": true, "description": "The crawler's product token — GPTBot, Googlebot, ClaudeBot — not a whole User-Agent header.", "schema": { "type": "string" } }, { "name": "path", "in": "query", "required": false, "description": "Path to test. Repeat for several, or send a comma separated list. Defaults to /. Max 200. Repeatable.", "schema": { "type": "array", "items": { "type": "string" } } }, { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } }, "post": { "summary": "Would this crawler fetch this path? (file as the raw body)", "operationId": "tool_robots_allowed_body", "tags": [ "tools" ], "description": "Identical to the GET, with the `robots_txt` argument taken from the raw request body instead of the query string — for files too large or too awkward to URL-encode. Still keyless.", "requestBody": { "required": true, "content": { "text/plain": { "schema": { "type": "string" } } } }, "responses": { "200": { "description": "The tool's answer.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "400": { "description": "The body was empty or unusable." } } } }, "/tools/ai-access": { "get": { "summary": "Which AI crawlers does this file actually stop?", "operationId": "tool_ai_access", "tags": [ "tools" ], "description": "Evaluate a pasted robots.txt against every AI crawler in this index and get the two lists that matter — blocked and allowed, per operator and category — plus the tokens in your file that match no known crawler (a typo blocks nothing), and the separation between crawlers that document obedience and the ones observed ignoring robots.txt, which need an IP or WAF rule instead.\n\nNo account, no key, no handshake. Called with no parameters at all it answers 200 with its own specification rather than 400, so walking /tools/ is free. This endpoint holds no copy of the logic: it calls the `audit_ai_access` MCP tool at https://www.pathwren.workers.dev/mcp/robots in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/ai-access.html · machine spec: https://www.pathwren.workers.dev/tools/ai-access.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/ai-access.html" }, "parameters": [ { "name": "robots_txt", "in": "query", "required": true, "description": "The contents of the robots.txt file, URL-encoded, or the raw POST body.", "schema": { "type": "string" } }, { "name": "path", "in": "query", "required": false, "description": "Path to test the verdict at. Defaults to /.", "schema": { "type": "string" } }, { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } }, "post": { "summary": "Which AI crawlers does this file actually stop? (file as the raw body)", "operationId": "tool_ai_access_body", "tags": [ "tools" ], "description": "Identical to the GET, with the `robots_txt` argument taken from the raw request body instead of the query string — for files too large or too awkward to URL-encode. Still keyless.", "requestBody": { "required": true, "content": { "text/plain": { "schema": { "type": "string" } } } }, "responses": { "200": { "description": "The tool's answer.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "400": { "description": "The body was empty or unusable." } } } }, "/tools/classify-ua": { "get": { "summary": "What is this user-agent?", "operationId": "tool_classify_ua", "tags": [ "tools" ], "description": "Identify the crawler behind a raw User-Agent header: operator, category, robots.txt token and stance, how it can be verified, and what blocking it costs. An unmatched string is reported as unmatched, with a labelled substring heuristic and never an identification.\n\nNo account, no key, no handshake. Called with no parameters at all it answers 200 with its own specification rather than 400, so walking /tools/ is free. This endpoint holds no copy of the logic: it calls the `classify_user_agent` MCP tool at https://www.pathwren.workers.dev/mcp in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/classify-ua.html · machine spec: https://www.pathwren.workers.dev/tools/classify-ua.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/classify-ua.html" }, "parameters": [ { "name": "ua", "in": "query", "required": true, "description": "The raw User-Agent header value, URL-encoded.", "schema": { "type": "string" } }, { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } } }, "/tools/verification-methods": { "get": { "summary": "Can this crawler be verified at all, and how?", "operationId": "tool_verification_methods", "tags": [ "tools" ], "description": "Before you read anything into a miss from /tools/verify-crawler: which crawlers can be verified by published prefix, which document forward-confirmed reverse DNS instead (with the exact two dig commands to run), and which publish no verification method at all — with the mirror's own counts and fetch dates.\n\nNo account, no key, no handshake. Called with no parameters at all it answers 200 with its own specification rather than 400, so walking /tools/ is free. This endpoint holds no copy of the logic: it calls the `verification_methods` MCP tool at https://www.pathwren.workers.dev/mcp/netcheck in-process, through that server's own JSON-RPC envelope. One implementation, two doors.\n\nDocs: https://www.pathwren.workers.dev/tools/verification-methods.html · machine spec: https://www.pathwren.workers.dev/tools/verification-methods.json", "externalDocs": { "description": "Worked examples and caveats", "url": "https://www.pathwren.workers.dev/tools/verification-methods.html" }, "parameters": [ { "name": "crawler", "in": "query", "required": false, "description": "Crawler slug, name, operator or user-agent substring. Omit for the whole tally.", "schema": { "type": "string" } }, { "name": "s", "in": "query", "required": false, "description": "Attribution tag. Not used to compute the answer: it tells this host which of its surfaces sent you, which is how it decides what to keep building. Leave the default in place if you have no reason not to.", "schema": { "type": "string", "default": "client-dossiers" } } ], "responses": { "200": { "description": "The tool's answer, or its specification if you sent no parameters.", "content": { "application/json": { "schema": { "type": "object" } } }, "links": { "changes": { "operationId": "getChanges", "description": "What changed since your cursor. Poll this instead of re-downloading this document: send the `cursor` from your last response back as `since`, and an answer with nothing in it costs about 2.5 KB (or a 304 with no body if you send back the ETag). The cursor advances only on a real change." } }, "headers": { "Link": { "description": "RFC 8288 links: the changes feed, a sibling document and the OpenAPI description of this host.", "schema": { "type": "string" }, "example": "; rel=\"related\"; title=\"changes feed: what moved since your cursor\"" } } }, "304": { "description": "Unchanged since the ETag you sent. No body." }, "400": { "description": "A required parameter is missing or unusable. The body names it and carries a worked example (RFC 9457)." }, "405": { "description": "Not a read method. Carries Allow and the MCP endpoint that does take a POST." } } } } }, "components": { "schemas": { "Change": { "type": "object", "description": "One observed change. `kind` distinguishes what really happened: ip_ranges_changed means the prefix SET moved and the added/removed lists are the real difference; upstream_republished means the operator reissued the same prefixes, which is not a reason to re-download; source_failed means an upstream stopped answering and its last good prefixes are still being served, marked stale.", "required": [ "seq", "at", "kind" ], "properties": { "seq": { "type": "integer", "description": "Monotonic cursor." }, "at": { "type": "string", "format": "date-time", "description": "When WE observed it — the fetch that saw the change." }, "kind": { "type": "string", "enum": [ "ip_ranges_changed", "upstream_republished", "source_failed", "source_recovered", "source_added", "source_removed", "crawler_added", "crawler_changed", "crawler_removed", "baseline" ] }, "title": { "type": "string" }, "detail": { "type": "string" }, "source": { "type": "string", "description": "Slug of the IP-range source." }, "crawler": { "type": "string", "description": "Slug of the crawler record." }, "operator_published_at": { "type": "string", "description": "The operator's own creationTime, carried through untouched." }, "added_count": { "type": "integer" }, "removed_count": { "type": "integer" }, "ipv4_added": { "type": "array", "items": { "type": "string" } }, "ipv4_removed": { "type": "array", "items": { "type": "string" } }, "ipv6_added": { "type": "array", "items": { "type": "string" } }, "ipv6_removed": { "type": "array", "items": { "type": "string" } }, "lists_truncated": { "type": "boolean", "description": "Counts are always exact; when true the prefix LISTS are a sample and the full set is in the source document." }, "affects": { "type": "array", "items": { "type": "string" }, "description": "The documents this change moved." } } }, "Crawler": { "type": "object", "required": [ "slug", "name", "operator", "category", "robots_token" ], "properties": { "slug": { "type": "string", "description": "Stable identifier used in URLs." }, "name": { "type": "string" }, "operator": { "type": "string" }, "operator_slug": { "type": "string" }, "category": { "type": "string", "enum": [ "ai-search", "ai-training", "archive", "dataset", "preview", "search", "seo", "tool", "user-fetch" ] }, "robots_token": { "type": "string", "description": "Exact User-agent value for robots.txt." }, "user_agent_substring": { "type": "string", "description": "Substring that reliably identifies it in a UA header. A match is a claim, not a proof." }, "user_agent_example": { "type": "string" }, "respects_robots_txt": { "type": "string", "enum": [ "documented", "by-design-no", "disputed", "n-a" ] }, "verification_method": { "type": "string", "enum": [ "published-ranges", "reverse-dns", "none" ] }, "published_ip_ranges_url": { "type": [ "string", "null" ], "format": "uri" }, "ipv4_prefix_count": { "type": "integer" }, "ipv6_prefix_count": { "type": "integer" }, "what_it_is": { "type": "string" }, "cost_of_blocking": { "type": "string", "description": "What you lose by disallowing it. This index's own assessment, not the operator's." }, "operator_docs": { "type": "string", "format": "uri" }, "last_reviewed": { "type": "string", "format": "date" } } } } } }