{ "category": "user-fetch", "label": "User-triggered fetchers", "description": "Fetch one page because a person asked for it, right then. One human intent, one request. Blocking them produces a visible error for a real reader.", "count": 12, "crawlers": [ { "slug": "chatgpt-agent", "name": "ChatGPT Agent", "operator": "OpenAI", "operator_slug": "openai", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "ChatGPT-User", "user_agent_substring": "ChatGPT Agent", "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/chatgpt-user.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json", "ipv4_prefix_count": 204, "ipv6_prefix_count": 0, "what_it_is": "ChatGPT's agent mode driving a real browser: it navigates and interacts with sites to finish a multi-step task a user gave it. OpenAI governs it with the ChatGPT-User token and the ChatGPT-User prefix list rather than a token of its own, so the robots rule and the address check are the same ones.", "cost_of_blocking": "Agentic tasks a user asked for — booking, comparing, filling a form on your site — fail. This is the fetch that ends in a transaction, so it is the most expensive user-triggered block on this list.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.html", "json_url": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.json", "last_reviewed": "2026-09-03" }, { "slug": "chatgpt-user", "name": "ChatGPT-User", "operator": "OpenAI", "operator_slug": "openai", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "ChatGPT-User", "user_agent_substring": "ChatGPT-User", "user_agent_example": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://openai.com/chatgpt-user.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/openai-chatgpt-user.json", "ipv4_prefix_count": 204, "ipv6_prefix_count": 0, "what_it_is": "Fetches a single page at the moment a user or a ChatGPT agent asks for it — a pasted link, a browsing step, an Operator task. One human intent, one request. OpenAI states these fetches are not used for training.", "cost_of_blocking": "ChatGPT cannot open your pages when a user explicitly asks it to. The user sees a fetch failure. This is usually the last bot anyone means to block.", "operator_docs": "https://platform.openai.com/docs/bots", "html_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/chatgpt-user.json", "last_reviewed": "2026-09-03" }, { "slug": "claude-user", "name": "Claude-User", "operator": "Anthropic", "operator_slug": "anthropic", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Claude-User", "user_agent_substring": "Claude-User", "user_agent_example": "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page because a Claude user asked Claude to read it, at that moment.", "cost_of_blocking": "Claude reports a fetch failure to a user who asked for your page by name.", "operator_docs": "https://support.anthropic.com/en/articles/8896518", "html_url": "https://www.pathwren.workers.dev/crawler/claude-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/claude-user.json", "last_reviewed": "2026-09-03" }, { "slug": "cohere-ai", "name": "cohere-ai", "operator": "Cohere", "operator_slug": "cohere", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "cohere-ai", "user_agent_substring": "cohere-ai", "user_agent_example": "Mozilla/5.0 (compatible; cohere-ai)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Cohere's fetcher, used when its assistant products need a page.", "cost_of_blocking": "Cohere-powered assistants cannot read your pages on request.", "operator_docs": "https://cohere.com/", "html_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.html", "json_url": "https://www.pathwren.workers.dev/crawler/cohere-ai.json", "last_reviewed": "2026-09-03" }, { "slug": "google-agent", "name": "Google-Agent", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-Agent", "user_agent_substring": "Google-Agent", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.json", "ipv4_prefix_count": 3, "ipv6_prefix_count": 1, "what_it_is": "Agents hosted on Google infrastructure navigating the web and taking actions on a user's request. Google names one prefix list for it — user-triggered-agents.json — and is separately experimenting with Web Bot Auth under the identity https://agent.bot.goog.", "cost_of_blocking": "Google-hosted agents cannot complete a task on your site for a user who asked them to. This is the agentic-commerce fetch: blocking it removes you from what an assistant can actually do rather than from what it can say.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-agent.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-agent.json", "last_reviewed": "2026-09-03" }, { "slug": "google-gemininotebook", "name": "Google-GeminiNotebook", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-GeminiNotebook", "user_agent_substring": "Google-GeminiNotebook", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches a URL a Gemini Notebook (formerly NotebookLM) user added as a source to their notebook. The former agent string Google-NotebookLM is documented as supported until August 2026. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "A user who deliberately added your page as a research source gets nothing. This is a citation-shaped fetch, not a training crawl.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.json", "last_reviewed": "2026-09-03" }, { "slug": "google-pinpoint", "name": "Google-Pinpoint", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-Pinpoint", "user_agent_substring": "Google-Pinpoint", "user_agent_example": "Google-Pinpoint", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches individual URLs that a Pinpoint user — usually a journalist or researcher — added as a source to their own document collection. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "A researcher who explicitly added your page to a collection cannot load it.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-pinpoint.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-pinpoint.json", "last_reviewed": "2026-09-03" }, { "slug": "google-read-aloud", "name": "Google-Read-Aloud", "operator": "Google", "operator_slug": "google", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Google-Read-Aloud", "user_agent_substring": "Google-Read-Aloud", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json", "ipv4_prefix_count": 529, "ipv6_prefix_count": 529, "what_it_is": "Fetches a page so Google can read it out loud with text-to-speech, at the moment a user asks. Formerly google-speakr. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.", "cost_of_blocking": "A reader who asked Google to read your page aloud — often someone using it for accessibility — gets an error instead.", "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "html_url": "https://www.pathwren.workers.dev/crawler/google-read-aloud.html", "json_url": "https://www.pathwren.workers.dev/crawler/google-read-aloud.json", "last_reviewed": "2026-09-03" }, { "slug": "meta-externalfetcher", "name": "meta-externalfetcher", "operator": "Meta", "operator_slug": "meta", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "meta-externalfetcher", "user_agent_substring": "meta-externalfetcher", "user_agent_example": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page when a Meta AI user asks about a specific link.", "cost_of_blocking": "Meta AI cannot read pages users hand it.", "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "html_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.html", "json_url": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.json", "last_reviewed": "2026-09-03" }, { "slug": "mistralai-user", "name": "MistralAI-User", "operator": "Mistral AI", "operator_slug": "mistral", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "MistralAI-User", "user_agent_substring": "MistralAI-User", "user_agent_example": "Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "none", "verification_label": "no published verification method", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page when a Le Chat user asks Mistral's assistant to read it.", "cost_of_blocking": "Le Chat cannot open links your readers give it.", "operator_docs": "https://docs.mistral.ai/", "html_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/mistralai-user.json", "last_reviewed": "2026-09-03" }, { "slug": "perplexity-user", "name": "Perplexity-User", "operator": "Perplexity", "operator_slug": "perplexity", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "Perplexity-User", "user_agent_substring": "Perplexity-User", "user_agent_example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "published-ranges", "verification_label": "published IP ranges", "published_ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json", "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/perplexity-user.json", "ipv4_prefix_count": 4, "ipv6_prefix_count": 0, "what_it_is": "Fetches a page because a Perplexity user asked for it. Perplexity documents that this fetch is user-initiated and is therefore not governed by robots.txt — a robots rule will not stop it, by stated policy.", "cost_of_blocking": "Not controllable via robots.txt. If you must stop it, verify by the published IP ranges and block at the edge — and accept that users who ask for your page get an error.", "operator_docs": "https://docs.perplexity.ai/guides/bots", "html_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.html", "json_url": "https://www.pathwren.workers.dev/crawler/perplexity-user.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexcalendar", "name": "YandexCalendar", "operator": "Yandex", "operator_slug": "yandex", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "YandexCalendar", "user_agent_substring": "YandexCalendar", "user_agent_example": "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Downloads calendar files a user subscribed to. Yandex notes these files are often in directories that are disallowed for indexing, which is why the general rules are not applied.", "cost_of_blocking": "Users who subscribed to a calendar you publish stop receiving updates.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexcalendar.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexcalendar.json", "last_reviewed": "2026-09-03" } ] }