{
 "operator": "Google",
 "slug": "google",
 "docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
 "crawler_count": 26,
 "crawlers": [
  {
   "slug": "adsbot-google",
   "name": "AdsBot-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "AdsBot-Google",
   "user_agent_substring": "AdsBot-Google",
   "user_agent_example": "AdsBot-Google (+http://www.google.com/adsbot.html)",
   "respects_robots_txt": "own-token-only",
   "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "Checks the quality of desktop landing pages for Google Ads. Google documents that it ignores the robots.txt * group with the ad publisher's permission, and obeys a group named for its own token.",
   "cost_of_blocking": "Google Ads cannot score your landing pages, which lowers Ad Rank on the ads pointing at them. If you do not buy ads, blocking it costs nothing but bandwidth savings.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/adsbot-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/adsbot-google.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "adsbot-google-mobile",
   "name": "AdsBot-Google-Mobile",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "AdsBot-Google-Mobile",
   "user_agent_substring": "AdsBot-Google-Mobile",
   "user_agent_example": "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
   "respects_robots_txt": "own-token-only",
   "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "The mobile-web landing page checker for Google Ads. Same rules as AdsBot-Google: the * group does not apply to it, its own token does.",
   "cost_of_blocking": "Mobile ad landing pages go unscored and the ads pointing at them rank worse. No effect on organic search.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "adsbot-google-mobile-apps",
   "name": "AdsBot-Google-Mobile-Apps",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "AdsBot-Google-Mobile-Apps",
   "user_agent_substring": "AdsBot-Google-Mobile-Apps",
   "user_agent_example": "AdsBot-Google-Mobile-Apps",
   "respects_robots_txt": "own-token-only",
   "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "Checks Android app landing pages for Google Ads. It obeys a group named for its own token and, per Google, follows the AdsBot-Google rules otherwise.",
   "cost_of_blocking": "App-install ad landing pages go unscored. Nothing organic changes.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "apis-google",
   "name": "APIs-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "APIs-Google",
   "user_agent_substring": "APIs-Google",
   "user_agent_example": "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)",
   "respects_robots_txt": "own-token-only",
   "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "Delivers push notifications for Google APIs to a webhook you registered. It is a special-case crawler: it ignores the robots.txt * group, because the fetch is a delivery to an address you asked it to deliver to.",
   "cost_of_blocking": "Google API push notifications stop arriving at your endpoint. This only affects services you set up yourself; there is no search or AI consequence.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/apis-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/apis-google.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "feedfetcher-google",
   "name": "FeedFetcher-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "FeedFetcher-Google",
   "user_agent_substring": "FeedFetcher-Google",
   "user_agent_example": "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Crawls RSS and Atom feeds for Google News and WebSub. It is a user-triggered fetcher, and Google documents that those generally ignore robots.txt because a person asked for the fetch. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "Feed-driven Google products stop seeing your updates. A robots.txt rule will not stop it — block by user-agent at the edge if you mean it.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-agent",
   "name": "Google-Agent",
   "operator": "Google",
   "operator_slug": "google",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Google-Agent",
   "user_agent_substring": "Google-Agent",
   "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.json",
   "ipv4_prefix_count": 3,
   "ipv6_prefix_count": 1,
   "what_it_is": "Agents hosted on Google infrastructure navigating the web and taking actions on a user's request. Google names one prefix list for it — user-triggered-agents.json — and is separately experimenting with Web Bot Auth under the identity https://agent.bot.goog.",
   "cost_of_blocking": "Google-hosted agents cannot complete a task on your site for a user who asked them to. This is the agentic-commerce fetch: blocking it removes you from what an assistant can actually do rather than from what it can say.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-agent.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-agent.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-cloudvertexbot",
   "name": "Google-CloudVertexBot",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-search",
   "category_label": "AI search crawlers",
   "robots_token": "Google-CloudVertexBot",
   "user_agent_substring": "Google-CloudVertexBot",
   "user_agent_example": "Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "Crawls a site on behalf of a Vertex AI Agent Builder customer who is building an agent over that site. It only visits sites the customer has asked it to.",
   "cost_of_blocking": "Third parties can no longer build Vertex AI agents that read your site. Irrelevant to Google Search.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-cws",
   "name": "Google-CWS",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Google-CWS",
   "user_agent_substring": "Google-CWS",
   "user_agent_example": "Mozilla/5.0 (compatible; Google-CWS)",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "The Chrome Web Store fetcher. It requests the URLs a developer put in the metadata of a Chrome extension or theme. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "Chrome Web Store listings that point at your pages cannot fetch them. Relevant only if you publish extensions.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-cws.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-cws.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-extended",
   "name": "Google-Extended",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "Google-Extended",
   "user_agent_substring": "(control token only — no crawler)",
   "user_agent_example": "(none: Google-Extended never appears as a user-agent)",
   "respects_robots_txt": "n-a",
   "respects_robots_txt_label": "control token only — no crawler",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Not a crawler. A robots.txt token that tells Google whether pages Googlebot already fetched may be used to train and ground Gemini. You will never see it in an access log; disallowing it changes what Google does with content it fetched under a different name.",
   "cost_of_blocking": "You are excluded from Gemini grounding and Gemini training. Google Search ranking and indexing are explicitly unaffected. This is the cleanest 'no training, keep my search traffic' lever that exists.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-extended.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-extended.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-gemininotebook",
   "name": "Google-GeminiNotebook",
   "operator": "Google",
   "operator_slug": "google",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Google-GeminiNotebook",
   "user_agent_substring": "Google-GeminiNotebook",
   "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Fetches a URL a Gemini Notebook (formerly NotebookLM) user added as a source to their notebook. The former agent string Google-NotebookLM is documented as supported until August 2026. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "A user who deliberately added your page as a research source gets nothing. This is a citation-shaped fetch, not a training crawl.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-inspectiontool",
   "name": "Google-InspectionTool",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Google-InspectionTool",
   "user_agent_substring": "Google-InspectionTool",
   "user_agent_example": "Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "The fetcher behind Search Console's URL Inspection and the Rich Results Test. It runs when a site owner clicks a button.",
   "cost_of_blocking": "Your own Search Console live tests stop working. Blocking this only hurts you.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-pinpoint",
   "name": "Google-Pinpoint",
   "operator": "Google",
   "operator_slug": "google",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Google-Pinpoint",
   "user_agent_substring": "Google-Pinpoint",
   "user_agent_example": "Google-Pinpoint",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Fetches individual URLs that a Pinpoint user — usually a journalist or researcher — added as a source to their own document collection. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "A researcher who explicitly added your page to a collection cannot load it.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-pinpoint.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-pinpoint.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-read-aloud",
   "name": "Google-Read-Aloud",
   "operator": "Google",
   "operator_slug": "google",
   "category": "user-fetch",
   "category_label": "User-triggered fetchers",
   "robots_token": "Google-Read-Aloud",
   "user_agent_substring": "Google-Read-Aloud",
   "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Fetches a page so Google can read it out loud with text-to-speech, at the moment a user asks. Formerly google-speakr. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "A reader who asked Google to read your page aloud — often someone using it for accessibility — gets an error instead.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-read-aloud.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-read-aloud.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-safety",
   "name": "Google-Safety",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Google-Safety",
   "user_agent_substring": "Google-Safety",
   "user_agent_example": "Google-Safety",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Google's abuse-investigation fetcher: malware review, phishing reports and similar. Google documents that it ignores robots.txt entirely, and a robots.txt rule for it does nothing.",
   "cost_of_blocking": "Nothing you can control. The rule is ignored by design; listing the token is documentation, not enforcement.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-safety.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-safety.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "google-site-verification",
   "name": "Google-Site-Verification",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Google-Site-Verification",
   "user_agent_substring": "Google-Site-Verification",
   "user_agent_example": "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Fetches the token file or meta tag that proves you own a site, when you click verify in Search Console. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "Your own Search Console verification fails. Blocking this only ever hurts the person doing the blocking.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/google-site-verification.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/google-site-verification.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googlebot",
   "name": "Googlebot",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot",
   "user_agent_substring": "Googlebot",
   "user_agent_example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 170,
   "ipv6_prefix_count": 147,
   "what_it_is": "The classic search crawler. It is also the crawler behind AI Overviews: Google does not run a separate bot for them, which is why the only AI opt-out is the Google-Extended token and not a Googlebot block.",
   "cost_of_blocking": "Total. You leave Google Search. Never block this to avoid AI use; use Google-Extended instead.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googlebot-image",
   "name": "Googlebot-Image",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-Image",
   "user_agent_substring": "Googlebot-Image",
   "user_agent_example": "Googlebot-Image/1.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 170,
   "ipv6_prefix_count": 147,
   "what_it_is": "Image indexing for Google Images. A separate token so you can leave images out of search without leaving search.",
   "cost_of_blocking": "Your images stop appearing in Google Images.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-image.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googlebot-news",
   "name": "Googlebot-News",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-News",
   "user_agent_substring": "Googlebot-News",
   "user_agent_example": "(uses the Googlebot user-agent; controlled by the Googlebot-News robots token)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 170,
   "ipv6_prefix_count": 147,
   "what_it_is": "A robots.txt token controlling inclusion in Google News. It does not have its own user-agent string; the fetch arrives as Googlebot.",
   "cost_of_blocking": "Removal from Google News, with normal Search unaffected.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-news.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googlebot-video",
   "name": "Googlebot-Video",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Googlebot-Video",
   "user_agent_substring": "Googlebot-Video",
   "user_agent_example": "Googlebot-Video/1.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json",
   "ipv4_prefix_count": 170,
   "ipv6_prefix_count": 147,
   "what_it_is": "The video half of Googlebot. It crawls video files and the pages around them for Google Video search, and it is matched by a robots.txt group for Googlebot as well as by its own token.",
   "cost_of_blocking": "Your videos leave Google video search. A rule for Googlebot already covers it, so blocking this token alone is usually a mistake of precision rather than of intent.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlebot-video.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlebot-video.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googlemessages",
   "name": "GoogleMessages",
   "operator": "Google",
   "operator_slug": "google",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "GoogleMessages",
   "user_agent_substring": "GoogleMessages",
   "user_agent_example": "GoogleMessages",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Generates the link preview when somebody sends one of your URLs in Google Messages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "Your links appear as bare URLs with no title or image in Google Messages chats. A preview fetcher is almost never the one you meant to block.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlemessages.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlemessages.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googleother",
   "name": "GoogleOther",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "GoogleOther",
   "user_agent_substring": "GoogleOther",
   "user_agent_example": "Mozilla/5.0 (compatible; GoogleOther)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "A generic fetcher used by Google product teams for one-off crawls and research, including data collection that does not belong to Search.",
   "cost_of_blocking": "No effect on Search indexing. Blocks internal Google research and product fetches.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googleother.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googleother.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googleother-image",
   "name": "GoogleOther-Image",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "GoogleOther-Image",
   "user_agent_substring": "GoogleOther-Image",
   "user_agent_example": "GoogleOther-Image/1.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "The image variant of GoogleOther: one-off fetches by Google product and research teams that are not Search. It also answers to a GoogleOther group in robots.txt.",
   "cost_of_blocking": "Google teams outside Search stop fetching your images. Image Search itself is unaffected — that is Googlebot-Image.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googleother-image.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googleother-image.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googleother-video",
   "name": "GoogleOther-Video",
   "operator": "Google",
   "operator_slug": "google",
   "category": "ai-training",
   "category_label": "AI training crawlers",
   "robots_token": "GoogleOther-Video",
   "user_agent_substring": "GoogleOther-Video",
   "user_agent_example": "GoogleOther-Video/1.0",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "The video variant of GoogleOther, used for internal Google fetches that do not belong to Search.",
   "cost_of_blocking": "No effect on Search or on Google Video search. Blocks internal Google research fetches of your video files.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googleother-video.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googleother-video.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googleproducer",
   "name": "GoogleProducer",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "GoogleProducer",
   "user_agent_substring": "GoogleProducer",
   "user_agent_example": "GoogleProducer; (+https://developers.google.com/search/docs/crawling-indexing/google-producer)",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Google Publisher Center: fetches the feeds a publisher explicitly supplied for Google News landing pages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "Your own Google News landing pages stop updating. Only publishers who configured Publisher Center are affected.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googleproducer.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googleproducer.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "mediapartners-google",
   "name": "Mediapartners-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "tool",
   "category_label": "Tools and frameworks",
   "robots_token": "Mediapartners-Google",
   "user_agent_substring": "Mediapartners-Google",
   "user_agent_example": "Mediapartners-Google",
   "respects_robots_txt": "own-token-only",
   "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "The AdSense crawler. It reads a page so AdSense can choose relevant ads for it, and it is a special-case crawler that ignores the robots.txt * group.",
   "cost_of_blocking": "Pages it cannot read get generic, lower-value AdSense ads or none at all. This is the one block on this list that costs you money directly if you run AdSense.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/mediapartners-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/mediapartners-google.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "storebot-google",
   "name": "Storebot-Google",
   "operator": "Google",
   "operator_slug": "google",
   "category": "search",
   "category_label": "Search engines",
   "robots_token": "Storebot-Google",
   "user_agent_substring": "Storebot-Google",
   "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; Storebot-Google/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-special.json",
   "ipv4_prefix_count": 136,
   "ipv6_prefix_count": 136,
   "what_it_is": "Checks shopping and checkout flows for Google's shopping surfaces.",
   "cost_of_blocking": "Product listings may lose shopping-specific enrichment. Irrelevant to non-commerce sites.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/storebot-google.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/storebot-google.json",
   "last_reviewed": "2026-09-03"
  }
 ],
 "ip_range_endpoints": [
  {
   "slug": "google-googlebot",
   "label": "Googlebot",
   "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/google-googlebot.json"
  },
  {
   "slug": "google-special",
   "label": "Google special-purpose crawlers",
   "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/google-special.json"
  },
  {
   "slug": "google-user-triggered",
   "label": "Google user-triggered fetchers",
   "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json"
  },
  {
   "slug": "google-user-triggered-google",
   "label": "Google user-triggered fetchers (Google-owned ranges)",
   "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-google.json"
  },
  {
   "slug": "google-user-triggered-agents",
   "label": "Google-Agent (agents on Google infrastructure)",
   "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json",
   "mirror": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered-agents.json"
  }
 ]
}