{
 "category": "preview",
 "label": "Link preview fetchers",
 "description": "Read your Open Graph tags when someone shares a link. Blocking these is almost always an accident.",
 "count": 4,
 "crawlers": [
  {
   "slug": "facebookexternalhit",
   "name": "facebookexternalhit",
   "operator": "Meta",
   "operator_slug": "meta",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "facebookexternalhit",
   "user_agent_substring": "facebookexternalhit",
   "user_agent_example": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The link unfurler: it reads your Open Graph tags when somebody shares your URL on a Meta property.",
   "cost_of_blocking": "Severe and usually accidental. Your links share as bare grey boxes with no title, image or description across Facebook, Instagram, Messenger and WhatsApp. Almost nobody means to block this.",
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "googlemessages",
   "name": "GoogleMessages",
   "operator": "Google",
   "operator_slug": "google",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "GoogleMessages",
   "user_agent_substring": "GoogleMessages",
   "user_agent_example": "GoogleMessages",
   "respects_robots_txt": "by-design-no",
   "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)",
   "verification_method": "published-ranges",
   "verification_label": "published IP ranges",
   "published_ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
   "ip_ranges_endpoint": "https://www.pathwren.workers.dev/ip-ranges/google-user-triggered.json",
   "ipv4_prefix_count": 529,
   "ipv6_prefix_count": 529,
   "what_it_is": "Generates the link preview when somebody sends one of your URLs in Google Messages. Google publishes fetcher addresses in two files — user-triggered-fetchers.json and user-triggered-fetchers-google.json — and does not say per fetcher which one applies, so verification means checking both; this index mirrors both.",
   "cost_of_blocking": "Your links appear as bare URLs with no title or image in Google Messages chats. A preview fetcher is almost never the one you meant to block.",
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
   "html_url": "https://www.pathwren.workers.dev/crawler/googlemessages.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/googlemessages.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "slackbot",
   "name": "Slackbot",
   "operator": "Slack",
   "operator_slug": "slack",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "Slackbot",
   "user_agent_substring": "Slackbot",
   "user_agent_example": "Slackbot 1.0 (+https://api.slack.com/robots)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "The other half of Slack's pair: the agent that reads robots.txt and handles Slack's non-unfurl fetches. Slack documents both strings on one page.",
   "cost_of_blocking": "Slack stops being able to read your robots.txt, which is a strange thing to want. Block the link expander instead if that is the goal.",
   "operator_docs": "https://api.slack.com/robots",
   "html_url": "https://www.pathwren.workers.dev/crawler/slackbot.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/slackbot.json",
   "last_reviewed": "2026-09-03"
  },
  {
   "slug": "slackbot-linkexpanding",
   "name": "Slackbot-LinkExpanding",
   "operator": "Slack",
   "operator_slug": "slack",
   "category": "preview",
   "category_label": "Link preview fetchers",
   "robots_token": "Slackbot-LinkExpanding",
   "user_agent_substring": "Slackbot-LinkExpanding",
   "user_agent_example": "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
   "respects_robots_txt": "documented",
   "respects_robots_txt_label": "obeys robots.txt (documented)",
   "verification_method": "none",
   "verification_label": "no published verification method",
   "published_ip_ranges_url": null,
   "ip_ranges_endpoint": null,
   "ipv4_prefix_count": 0,
   "ipv6_prefix_count": 0,
   "what_it_is": "Fetches a page to build the unfurl card when somebody pastes your link into Slack. One paste, one fetch.",
   "cost_of_blocking": "Your links appear in Slack as bare URLs. Inside working teams that quietly costs you clicks, and nothing is gained.",
   "operator_docs": "https://api.slack.com/robots",
   "html_url": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.html",
   "json_url": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.json",
   "last_reviewed": "2026-09-03"
  }
 ]
}