{
 "name": "All 150 crawlers",
 "description": "Every crawler in the index — 150 entries across 74 operators, each checked against the operator's own documentation. One page, one JSON record and one markdown file per crawler.",
 "url": "https://www.pathwren.workers.dev/crawler",
 "canonical": "https://www.pathwren.workers.dev/crawler/index.html",
 "formats": {
  "html": "https://www.pathwren.workers.dev/crawler/index.html",
  "json": "https://www.pathwren.workers.dev/crawler/index.json",
  "markdown": "https://www.pathwren.workers.dev/crawler/index.md"
 },
 "curl": "curl -s https://www.pathwren.workers.dev/data/agents.json | jq '.crawlers[].robots_token'",
 "count": 150,
 "entries": [
  {
   "slug": "adsbot-google",
   "name": "AdsBot-Google",
   "operator": "Google",
   "category": "tool",
   "robots_token": "AdsBot-Google",
   "html": "https://www.pathwren.workers.dev/crawler/adsbot-google.html",
   "json": "https://www.pathwren.workers.dev/crawler/adsbot-google.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/adsbot-google.md"
  },
  {
   "slug": "adsbot-google-mobile",
   "name": "AdsBot-Google-Mobile",
   "operator": "Google",
   "category": "tool",
   "robots_token": "AdsBot-Google-Mobile",
   "html": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.html",
   "json": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile.md"
  },
  {
   "slug": "adsbot-google-mobile-apps",
   "name": "AdsBot-Google-Mobile-Apps",
   "operator": "Google",
   "category": "tool",
   "robots_token": "AdsBot-Google-Mobile-Apps",
   "html": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.html",
   "json": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/adsbot-google-mobile-apps.md"
  },
  {
   "slug": "ahrefsbot",
   "name": "AhrefsBot",
   "operator": "Ahrefs",
   "category": "seo",
   "robots_token": "AhrefsBot",
   "html": "https://www.pathwren.workers.dev/crawler/ahrefsbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/ahrefsbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/ahrefsbot.md"
  },
  {
   "slug": "ahrefssiteaudit",
   "name": "AhrefsSiteAudit",
   "operator": "Ahrefs",
   "category": "seo",
   "robots_token": "AhrefsSiteAudit",
   "html": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.html",
   "json": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/ahrefssiteaudit.md"
  },
  {
   "slug": "ai2bot",
   "name": "AI2Bot",
   "operator": "Allen Institute for AI",
   "category": "dataset",
   "robots_token": "AI2Bot",
   "html": "https://www.pathwren.workers.dev/crawler/ai2bot.html",
   "json": "https://www.pathwren.workers.dev/crawler/ai2bot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/ai2bot.md"
  },
  {
   "slug": "ai2bot-dolma",
   "name": "Ai2Bot-Dolma",
   "operator": "Allen Institute for AI",
   "category": "dataset",
   "robots_token": "Ai2Bot-Dolma",
   "html": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.html",
   "json": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/ai2bot-dolma.md"
  },
  {
   "slug": "aihitbot",
   "name": "aiHitBot",
   "operator": "aiHit",
   "category": "dataset",
   "robots_token": "aiHitBot",
   "html": "https://www.pathwren.workers.dev/crawler/aihitbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/aihitbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/aihitbot.md"
  },
  {
   "slug": "aiwebindex",
   "name": "AIWebIndex",
   "operator": "Lyrenth",
   "category": "ai-search",
   "robots_token": "AIWebIndex",
   "html": "https://www.pathwren.workers.dev/crawler/aiwebindex.html",
   "json": "https://www.pathwren.workers.dev/crawler/aiwebindex.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/aiwebindex.md"
  },
  {
   "slug": "amazonbot",
   "name": "Amazonbot",
   "operator": "Amazon",
   "category": "ai-search",
   "robots_token": "Amazonbot",
   "html": "https://www.pathwren.workers.dev/crawler/amazonbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/amazonbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/amazonbot.md"
  },
  {
   "slug": "andibot",
   "name": "Andibot",
   "operator": "Andi",
   "category": "ai-search",
   "robots_token": "Andibot",
   "html": "https://www.pathwren.workers.dev/crawler/andibot.html",
   "json": "https://www.pathwren.workers.dev/crawler/andibot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/andibot.md"
  },
  {
   "slug": "anomura",
   "name": "Anomura",
   "operator": "Direqt",
   "category": "ai-search",
   "robots_token": "Anomura",
   "html": "https://www.pathwren.workers.dev/crawler/anomura.html",
   "json": "https://www.pathwren.workers.dev/crawler/anomura.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/anomura.md"
  },
  {
   "slug": "anthropic-ai",
   "name": "anthropic-ai",
   "operator": "Anthropic",
   "category": "ai-training",
   "robots_token": "anthropic-ai",
   "html": "https://www.pathwren.workers.dev/crawler/anthropic-ai.html",
   "json": "https://www.pathwren.workers.dev/crawler/anthropic-ai.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/anthropic-ai.md"
  },
  {
   "slug": "apis-google",
   "name": "APIs-Google",
   "operator": "Google",
   "category": "tool",
   "robots_token": "APIs-Google",
   "html": "https://www.pathwren.workers.dev/crawler/apis-google.html",
   "json": "https://www.pathwren.workers.dev/crawler/apis-google.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/apis-google.md"
  },
  {
   "slug": "applebot",
   "name": "Applebot",
   "operator": "Apple",
   "category": "search",
   "robots_token": "Applebot",
   "html": "https://www.pathwren.workers.dev/crawler/applebot.html",
   "json": "https://www.pathwren.workers.dev/crawler/applebot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/applebot.md"
  },
  {
   "slug": "applebot-extended",
   "name": "Applebot-Extended",
   "operator": "Apple",
   "category": "ai-training",
   "robots_token": "Applebot-Extended",
   "html": "https://www.pathwren.workers.dev/crawler/applebot-extended.html",
   "json": "https://www.pathwren.workers.dev/crawler/applebot-extended.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/applebot-extended.md"
  },
  {
   "slug": "archive-org-bot",
   "name": "archive.org_bot",
   "operator": "Internet Archive",
   "category": "archive",
   "robots_token": "archive.org_bot",
   "html": "https://www.pathwren.workers.dev/crawler/archive-org-bot.html",
   "json": "https://www.pathwren.workers.dev/crawler/archive-org-bot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/archive-org-bot.md"
  },
  {
   "slug": "atlassian-bot",
   "name": "atlassian-bot",
   "operator": "Atlassian",
   "category": "ai-search",
   "robots_token": "atlassian-bot",
   "html": "https://www.pathwren.workers.dev/crawler/atlassian-bot.html",
   "json": "https://www.pathwren.workers.dev/crawler/atlassian-bot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/atlassian-bot.md"
  },
  {
   "slug": "awariorssbot",
   "name": "AwarioRssBot",
   "operator": "Awario",
   "category": "dataset",
   "robots_token": "AwarioRssBot",
   "html": "https://www.pathwren.workers.dev/crawler/awariorssbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/awariorssbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/awariorssbot.md"
  },
  {
   "slug": "awariosmartbot",
   "name": "AwarioSmartBot",
   "operator": "Awario",
   "category": "dataset",
   "robots_token": "AwarioSmartBot",
   "html": "https://www.pathwren.workers.dev/crawler/awariosmartbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/awariosmartbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/awariosmartbot.md"
  },
  {
   "slug": "baiduspider",
   "name": "Baiduspider",
   "operator": "Baidu",
   "category": "search",
   "robots_token": "Baiduspider",
   "html": "https://www.pathwren.workers.dev/crawler/baiduspider.html",
   "json": "https://www.pathwren.workers.dev/crawler/baiduspider.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/baiduspider.md"
  },
  {
   "slug": "barkrowler",
   "name": "Barkrowler",
   "operator": "Babbar",
   "category": "seo",
   "robots_token": "barkrowler",
   "html": "https://www.pathwren.workers.dev/crawler/barkrowler.html",
   "json": "https://www.pathwren.workers.dev/crawler/barkrowler.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/barkrowler.md"
  },
  {
   "slug": "bedrockbot",
   "name": "bedrockbot",
   "operator": "Amazon",
   "category": "ai-search",
   "robots_token": "bedrockbot",
   "html": "https://www.pathwren.workers.dev/crawler/bedrockbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/bedrockbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/bedrockbot.md"
  },
  {
   "slug": "bingbot",
   "name": "bingbot",
   "operator": "Microsoft",
   "category": "search",
   "robots_token": "bingbot",
   "html": "https://www.pathwren.workers.dev/crawler/bingbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/bingbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/bingbot.md"
  },
  {
   "slug": "bytespider",
   "name": "Bytespider",
   "operator": "ByteDance",
   "category": "ai-training",
   "robots_token": "Bytespider",
   "html": "https://www.pathwren.workers.dev/crawler/bytespider.html",
   "json": "https://www.pathwren.workers.dev/crawler/bytespider.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/bytespider.md"
  },
  {
   "slug": "ccbot",
   "name": "CCBot",
   "operator": "Common Crawl",
   "category": "dataset",
   "robots_token": "CCBot",
   "html": "https://www.pathwren.workers.dev/crawler/ccbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/ccbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/ccbot.md"
  },
  {
   "slug": "chatgpt-agent",
   "name": "ChatGPT Agent",
   "operator": "OpenAI",
   "category": "user-fetch",
   "robots_token": "ChatGPT-User",
   "html": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.html",
   "json": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/chatgpt-agent.md"
  },
  {
   "slug": "chatgpt-user",
   "name": "ChatGPT-User",
   "operator": "OpenAI",
   "category": "user-fetch",
   "robots_token": "ChatGPT-User",
   "html": "https://www.pathwren.workers.dev/crawler/chatgpt-user.html",
   "json": "https://www.pathwren.workers.dev/crawler/chatgpt-user.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/chatgpt-user.md"
  },
  {
   "slug": "claude-searchbot",
   "name": "Claude-SearchBot",
   "operator": "Anthropic",
   "category": "ai-search",
   "robots_token": "Claude-SearchBot",
   "html": "https://www.pathwren.workers.dev/crawler/claude-searchbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/claude-searchbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/claude-searchbot.md"
  },
  {
   "slug": "claude-user",
   "name": "Claude-User",
   "operator": "Anthropic",
   "category": "user-fetch",
   "robots_token": "Claude-User",
   "html": "https://www.pathwren.workers.dev/crawler/claude-user.html",
   "json": "https://www.pathwren.workers.dev/crawler/claude-user.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/claude-user.md"
  },
  {
   "slug": "claude-web",
   "name": "Claude-Web",
   "operator": "Anthropic",
   "category": "ai-search",
   "robots_token": "Claude-Web",
   "html": "https://www.pathwren.workers.dev/crawler/claude-web.html",
   "json": "https://www.pathwren.workers.dev/crawler/claude-web.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/claude-web.md"
  },
  {
   "slug": "claudebot",
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "category": "ai-training",
   "robots_token": "ClaudeBot",
   "html": "https://www.pathwren.workers.dev/crawler/claudebot.html",
   "json": "https://www.pathwren.workers.dev/crawler/claudebot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/claudebot.md"
  },
  {
   "slug": "cloudflare-autorag",
   "name": "Cloudflare-AutoRAG",
   "operator": "Cloudflare",
   "category": "ai-search",
   "robots_token": "Cloudflare-AutoRAG",
   "html": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.html",
   "json": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/cloudflare-autorag.md"
  },
  {
   "slug": "cohere-ai",
   "name": "cohere-ai",
   "operator": "Cohere",
   "category": "user-fetch",
   "robots_token": "cohere-ai",
   "html": "https://www.pathwren.workers.dev/crawler/cohere-ai.html",
   "json": "https://www.pathwren.workers.dev/crawler/cohere-ai.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/cohere-ai.md"
  },
  {
   "slug": "cohere-training-data-crawler",
   "name": "cohere-training-data-crawler",
   "operator": "Cohere",
   "category": "ai-training",
   "robots_token": "cohere-training-data-crawler",
   "html": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.html",
   "json": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/cohere-training-data-crawler.md"
  },
  {
   "slug": "cotoyogi",
   "name": "Cotoyogi",
   "operator": "ROIS-DS",
   "category": "ai-training",
   "robots_token": "Cotoyogi",
   "html": "https://www.pathwren.workers.dev/crawler/cotoyogi.html",
   "json": "https://www.pathwren.workers.dev/crawler/cotoyogi.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/cotoyogi.md"
  },
  {
   "slug": "crawl4ai",
   "name": "Crawl4AI",
   "operator": "Crawl4AI project",
   "category": "tool",
   "robots_token": "Crawl4AI",
   "html": "https://www.pathwren.workers.dev/crawler/crawl4ai.html",
   "json": "https://www.pathwren.workers.dev/crawler/crawl4ai.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/crawl4ai.md"
  },
  {
   "slug": "crawlspace",
   "name": "Crawlspace",
   "operator": "Crawlspace",
   "category": "tool",
   "robots_token": "Crawlspace",
   "html": "https://www.pathwren.workers.dev/crawler/crawlspace.html",
   "json": "https://www.pathwren.workers.dev/crawler/crawlspace.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/crawlspace.md"
  },
  {
   "slug": "dataforseobot",
   "name": "DataForSeoBot",
   "operator": "DataForSEO",
   "category": "seo",
   "robots_token": "DataForSeoBot",
   "html": "https://www.pathwren.workers.dev/crawler/dataforseobot.html",
   "json": "https://www.pathwren.workers.dev/crawler/dataforseobot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/dataforseobot.md"
  },
  {
   "slug": "diffbot",
   "name": "Diffbot",
   "operator": "Diffbot",
   "category": "dataset",
   "robots_token": "Diffbot",
   "html": "https://www.pathwren.workers.dev/crawler/diffbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/diffbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/diffbot.md"
  },
  {
   "slug": "dotbot",
   "name": "DotBot",
   "operator": "Moz",
   "category": "seo",
   "robots_token": "dotbot",
   "html": "https://www.pathwren.workers.dev/crawler/dotbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/dotbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/dotbot.md"
  },
  {
   "slug": "duckassistbot",
   "name": "DuckAssistBot",
   "operator": "DuckDuckGo",
   "category": "ai-search",
   "robots_token": "DuckAssistBot",
   "html": "https://www.pathwren.workers.dev/crawler/duckassistbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/duckassistbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/duckassistbot.md"
  },
  {
   "slug": "duckduckbot",
   "name": "DuckDuckBot",
   "operator": "DuckDuckGo",
   "category": "search",
   "robots_token": "DuckDuckBot",
   "html": "https://www.pathwren.workers.dev/crawler/duckduckbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/duckduckbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/duckduckbot.md"
  },
  {
   "slug": "echoboxbot",
   "name": "EchoboxBot",
   "operator": "Echobox",
   "category": "dataset",
   "robots_token": "EchoboxBot",
   "html": "https://www.pathwren.workers.dev/crawler/echoboxbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/echoboxbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/echoboxbot.md"
  },
  {
   "slug": "exasearchbot",
   "name": "ExaSearchBot",
   "operator": "Exa",
   "category": "ai-search",
   "robots_token": "ExaSearchBot",
   "html": "https://www.pathwren.workers.dev/crawler/exasearchbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/exasearchbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/exasearchbot.md"
  },
  {
   "slug": "facebookbot",
   "name": "FacebookBot",
   "operator": "Meta",
   "category": "ai-training",
   "robots_token": "FacebookBot",
   "html": "https://www.pathwren.workers.dev/crawler/facebookbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/facebookbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/facebookbot.md"
  },
  {
   "slug": "facebookexternalhit",
   "name": "facebookexternalhit",
   "operator": "Meta",
   "category": "preview",
   "robots_token": "facebookexternalhit",
   "html": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.html",
   "json": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/facebookexternalhit.md"
  },
  {
   "slug": "factset-spyderbot",
   "name": "Factset_spyderbot",
   "operator": "FactSet",
   "category": "ai-training",
   "robots_token": "Factset_spyderbot",
   "html": "https://www.pathwren.workers.dev/crawler/factset-spyderbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/factset-spyderbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/factset-spyderbot.md"
  },
  {
   "slug": "feedfetcher-google",
   "name": "FeedFetcher-Google",
   "operator": "Google",
   "category": "tool",
   "robots_token": "FeedFetcher-Google",
   "html": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.html",
   "json": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/feedfetcher-google.md"
  },
  {
   "slug": "firecrawlagent",
   "name": "FirecrawlAgent",
   "operator": "Firecrawl",
   "category": "tool",
   "robots_token": "FirecrawlAgent",
   "html": "https://www.pathwren.workers.dev/crawler/firecrawlagent.html",
   "json": "https://www.pathwren.workers.dev/crawler/firecrawlagent.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/firecrawlagent.md"
  },
  {
   "slug": "google-agent",
   "name": "Google-Agent",
   "operator": "Google",
   "category": "user-fetch",
   "robots_token": "Google-Agent",
   "html": "https://www.pathwren.workers.dev/crawler/google-agent.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-agent.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-agent.md"
  },
  {
   "slug": "google-cloudvertexbot",
   "name": "Google-CloudVertexBot",
   "operator": "Google",
   "category": "ai-search",
   "robots_token": "Google-CloudVertexBot",
   "html": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-cloudvertexbot.md"
  },
  {
   "slug": "google-cws",
   "name": "Google-CWS",
   "operator": "Google",
   "category": "tool",
   "robots_token": "Google-CWS",
   "html": "https://www.pathwren.workers.dev/crawler/google-cws.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-cws.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-cws.md"
  },
  {
   "slug": "google-extended",
   "name": "Google-Extended",
   "operator": "Google",
   "category": "ai-training",
   "robots_token": "Google-Extended",
   "html": "https://www.pathwren.workers.dev/crawler/google-extended.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-extended.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-extended.md"
  },
  {
   "slug": "google-gemininotebook",
   "name": "Google-GeminiNotebook",
   "operator": "Google",
   "category": "user-fetch",
   "robots_token": "Google-GeminiNotebook",
   "html": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-gemininotebook.md"
  },
  {
   "slug": "google-inspectiontool",
   "name": "Google-InspectionTool",
   "operator": "Google",
   "category": "tool",
   "robots_token": "Google-InspectionTool",
   "html": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-inspectiontool.md"
  },
  {
   "slug": "google-pinpoint",
   "name": "Google-Pinpoint",
   "operator": "Google",
   "category": "user-fetch",
   "robots_token": "Google-Pinpoint",
   "html": "https://www.pathwren.workers.dev/crawler/google-pinpoint.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-pinpoint.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-pinpoint.md"
  },
  {
   "slug": "google-read-aloud",
   "name": "Google-Read-Aloud",
   "operator": "Google",
   "category": "user-fetch",
   "robots_token": "Google-Read-Aloud",
   "html": "https://www.pathwren.workers.dev/crawler/google-read-aloud.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-read-aloud.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-read-aloud.md"
  },
  {
   "slug": "google-safety",
   "name": "Google-Safety",
   "operator": "Google",
   "category": "tool",
   "robots_token": "Google-Safety",
   "html": "https://www.pathwren.workers.dev/crawler/google-safety.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-safety.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-safety.md"
  },
  {
   "slug": "google-site-verification",
   "name": "Google-Site-Verification",
   "operator": "Google",
   "category": "tool",
   "robots_token": "Google-Site-Verification",
   "html": "https://www.pathwren.workers.dev/crawler/google-site-verification.html",
   "json": "https://www.pathwren.workers.dev/crawler/google-site-verification.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/google-site-verification.md"
  },
  {
   "slug": "googlebot",
   "name": "Googlebot",
   "operator": "Google",
   "category": "search",
   "robots_token": "Googlebot",
   "html": "https://www.pathwren.workers.dev/crawler/googlebot.html",
   "json": "https://www.pathwren.workers.dev/crawler/googlebot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googlebot.md"
  },
  {
   "slug": "googlebot-image",
   "name": "Googlebot-Image",
   "operator": "Google",
   "category": "search",
   "robots_token": "Googlebot-Image",
   "html": "https://www.pathwren.workers.dev/crawler/googlebot-image.html",
   "json": "https://www.pathwren.workers.dev/crawler/googlebot-image.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googlebot-image.md"
  },
  {
   "slug": "googlebot-news",
   "name": "Googlebot-News",
   "operator": "Google",
   "category": "search",
   "robots_token": "Googlebot-News",
   "html": "https://www.pathwren.workers.dev/crawler/googlebot-news.html",
   "json": "https://www.pathwren.workers.dev/crawler/googlebot-news.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googlebot-news.md"
  },
  {
   "slug": "googlebot-video",
   "name": "Googlebot-Video",
   "operator": "Google",
   "category": "search",
   "robots_token": "Googlebot-Video",
   "html": "https://www.pathwren.workers.dev/crawler/googlebot-video.html",
   "json": "https://www.pathwren.workers.dev/crawler/googlebot-video.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googlebot-video.md"
  },
  {
   "slug": "googlemessages",
   "name": "GoogleMessages",
   "operator": "Google",
   "category": "preview",
   "robots_token": "GoogleMessages",
   "html": "https://www.pathwren.workers.dev/crawler/googlemessages.html",
   "json": "https://www.pathwren.workers.dev/crawler/googlemessages.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googlemessages.md"
  },
  {
   "slug": "googleother",
   "name": "GoogleOther",
   "operator": "Google",
   "category": "ai-training",
   "robots_token": "GoogleOther",
   "html": "https://www.pathwren.workers.dev/crawler/googleother.html",
   "json": "https://www.pathwren.workers.dev/crawler/googleother.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googleother.md"
  },
  {
   "slug": "googleother-image",
   "name": "GoogleOther-Image",
   "operator": "Google",
   "category": "ai-training",
   "robots_token": "GoogleOther-Image",
   "html": "https://www.pathwren.workers.dev/crawler/googleother-image.html",
   "json": "https://www.pathwren.workers.dev/crawler/googleother-image.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googleother-image.md"
  },
  {
   "slug": "googleother-video",
   "name": "GoogleOther-Video",
   "operator": "Google",
   "category": "ai-training",
   "robots_token": "GoogleOther-Video",
   "html": "https://www.pathwren.workers.dev/crawler/googleother-video.html",
   "json": "https://www.pathwren.workers.dev/crawler/googleother-video.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googleother-video.md"
  },
  {
   "slug": "googleproducer",
   "name": "GoogleProducer",
   "operator": "Google",
   "category": "tool",
   "robots_token": "GoogleProducer",
   "html": "https://www.pathwren.workers.dev/crawler/googleproducer.html",
   "json": "https://www.pathwren.workers.dev/crawler/googleproducer.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/googleproducer.md"
  },
  {
   "slug": "gptbot",
   "name": "GPTBot",
   "operator": "OpenAI",
   "category": "ai-training",
   "robots_token": "GPTBot",
   "html": "https://www.pathwren.workers.dev/crawler/gptbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/gptbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/gptbot.md"
  },
  {
   "slug": "ia-archiver",
   "name": "ia_archiver",
   "operator": "Internet Archive",
   "category": "archive",
   "robots_token": "ia_archiver",
   "html": "https://www.pathwren.workers.dev/crawler/ia-archiver.html",
   "json": "https://www.pathwren.workers.dev/crawler/ia-archiver.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/ia-archiver.md"
  },
  {
   "slug": "icc-crawler",
   "name": "ICC-Crawler",
   "operator": "NICT",
   "category": "ai-training",
   "robots_token": "ICC-Crawler",
   "html": "https://www.pathwren.workers.dev/crawler/icc-crawler.html",
   "json": "https://www.pathwren.workers.dev/crawler/icc-crawler.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/icc-crawler.md"
  },
  {
   "slug": "imagesiftbot",
   "name": "ImagesiftBot",
   "operator": "Hive AI",
   "category": "dataset",
   "robots_token": "ImagesiftBot",
   "html": "https://www.pathwren.workers.dev/crawler/imagesiftbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/imagesiftbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/imagesiftbot.md"
  },
  {
   "slug": "img2dataset",
   "name": "img2dataset",
   "operator": "LAION / img2dataset",
   "category": "dataset",
   "robots_token": "img2dataset",
   "html": "https://www.pathwren.workers.dev/crawler/img2dataset.html",
   "json": "https://www.pathwren.workers.dev/crawler/img2dataset.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/img2dataset.md"
  },
  {
   "slug": "isscyberriskcrawler",
   "name": "ISSCyberRiskCrawler",
   "operator": "ISS Corporate Solutions",
   "category": "ai-training",
   "robots_token": "ISSCyberRiskCrawler",
   "html": "https://www.pathwren.workers.dev/crawler/isscyberriskcrawler.html",
   "json": "https://www.pathwren.workers.dev/crawler/isscyberriskcrawler.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/isscyberriskcrawler.md"
  },
  {
   "slug": "kagibot",
   "name": "Kagibot",
   "operator": "Kagi",
   "category": "search",
   "robots_token": "Kagibot",
   "html": "https://www.pathwren.workers.dev/crawler/kagibot.html",
   "json": "https://www.pathwren.workers.dev/crawler/kagibot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/kagibot.md"
  },
  {
   "slug": "klaviyoaibot",
   "name": "KlaviyoAIBot",
   "operator": "Klaviyo",
   "category": "ai-search",
   "robots_token": "KlaviyoAIBot",
   "html": "https://www.pathwren.workers.dev/crawler/klaviyoaibot.html",
   "json": "https://www.pathwren.workers.dev/crawler/klaviyoaibot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/klaviyoaibot.md"
  },
  {
   "slug": "laiondownloader",
   "name": "LAIONDownloader",
   "operator": "LAION / img2dataset",
   "category": "dataset",
   "robots_token": "LAIONDownloader",
   "html": "https://www.pathwren.workers.dev/crawler/laiondownloader.html",
   "json": "https://www.pathwren.workers.dev/crawler/laiondownloader.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/laiondownloader.md"
  },
  {
   "slug": "lightpanda",
   "name": "Lightpanda",
   "operator": "Lightpanda",
   "category": "tool",
   "robots_token": "Lightpanda",
   "html": "https://www.pathwren.workers.dev/crawler/lightpanda.html",
   "json": "https://www.pathwren.workers.dev/crawler/lightpanda.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/lightpanda.md"
  },
  {
   "slug": "linguee-bot",
   "name": "Linguee Bot",
   "operator": "Linguee",
   "category": "ai-training",
   "robots_token": "Linguee Bot",
   "html": "https://www.pathwren.workers.dev/crawler/linguee-bot.html",
   "json": "https://www.pathwren.workers.dev/crawler/linguee-bot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/linguee-bot.md"
  },
  {
   "slug": "mediapartners-google",
   "name": "Mediapartners-Google",
   "operator": "Google",
   "category": "tool",
   "robots_token": "Mediapartners-Google",
   "html": "https://www.pathwren.workers.dev/crawler/mediapartners-google.html",
   "json": "https://www.pathwren.workers.dev/crawler/mediapartners-google.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/mediapartners-google.md"
  },
  {
   "slug": "meta-externalagent",
   "name": "meta-externalagent",
   "operator": "Meta",
   "category": "ai-training",
   "robots_token": "meta-externalagent",
   "html": "https://www.pathwren.workers.dev/crawler/meta-externalagent.html",
   "json": "https://www.pathwren.workers.dev/crawler/meta-externalagent.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/meta-externalagent.md"
  },
  {
   "slug": "meta-externalfetcher",
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "category": "user-fetch",
   "robots_token": "meta-externalfetcher",
   "html": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.html",
   "json": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/meta-externalfetcher.md"
  },
  {
   "slug": "meta-webindexer",
   "name": "Meta-WebIndexer",
   "operator": "Meta",
   "category": "ai-search",
   "robots_token": "Meta-WebIndexer",
   "html": "https://www.pathwren.workers.dev/crawler/meta-webindexer.html",
   "json": "https://www.pathwren.workers.dev/crawler/meta-webindexer.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/meta-webindexer.md"
  },
  {
   "slug": "mistralai-user",
   "name": "MistralAI-User",
   "operator": "Mistral AI",
   "category": "user-fetch",
   "robots_token": "MistralAI-User",
   "html": "https://www.pathwren.workers.dev/crawler/mistralai-user.html",
   "json": "https://www.pathwren.workers.dev/crawler/mistralai-user.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/mistralai-user.md"
  },
  {
   "slug": "mj12bot",
   "name": "MJ12bot",
   "operator": "Majestic",
   "category": "seo",
   "robots_token": "MJ12bot",
   "html": "https://www.pathwren.workers.dev/crawler/mj12bot.html",
   "json": "https://www.pathwren.workers.dev/crawler/mj12bot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/mj12bot.md"
  },
  {
   "slug": "mojeekbot",
   "name": "MojeekBot",
   "operator": "Mojeek",
   "category": "search",
   "robots_token": "MojeekBot",
   "html": "https://www.pathwren.workers.dev/crawler/mojeekbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/mojeekbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/mojeekbot.md"
  },
  {
   "slug": "oai-searchbot",
   "name": "OAI-SearchBot",
   "operator": "OpenAI",
   "category": "ai-search",
   "robots_token": "OAI-SearchBot",
   "html": "https://www.pathwren.workers.dev/crawler/oai-searchbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/oai-searchbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/oai-searchbot.md"
  },
  {
   "slug": "omgili",
   "name": "omgili",
   "operator": "Webz.io",
   "category": "dataset",
   "robots_token": "omgili",
   "html": "https://www.pathwren.workers.dev/crawler/omgili.html",
   "json": "https://www.pathwren.workers.dev/crawler/omgili.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/omgili.md"
  },
  {
   "slug": "omgilibot",
   "name": "omgilibot",
   "operator": "Webz.io",
   "category": "dataset",
   "robots_token": "omgilibot",
   "html": "https://www.pathwren.workers.dev/crawler/omgilibot.html",
   "json": "https://www.pathwren.workers.dev/crawler/omgilibot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/omgilibot.md"
  },
  {
   "slug": "panscient",
   "name": "Panscient",
   "operator": "Panscient",
   "category": "dataset",
   "robots_token": "panscient.com",
   "html": "https://www.pathwren.workers.dev/crawler/panscient.html",
   "json": "https://www.pathwren.workers.dev/crawler/panscient.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/panscient.md"
  },
  {
   "slug": "perplexity-user",
   "name": "Perplexity-User",
   "operator": "Perplexity",
   "category": "user-fetch",
   "robots_token": "Perplexity-User",
   "html": "https://www.pathwren.workers.dev/crawler/perplexity-user.html",
   "json": "https://www.pathwren.workers.dev/crawler/perplexity-user.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/perplexity-user.md"
  },
  {
   "slug": "perplexitybot",
   "name": "PerplexityBot",
   "operator": "Perplexity",
   "category": "ai-search",
   "robots_token": "PerplexityBot",
   "html": "https://www.pathwren.workers.dev/crawler/perplexitybot.html",
   "json": "https://www.pathwren.workers.dev/crawler/perplexitybot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/perplexitybot.md"
  },
  {
   "slug": "petalbot",
   "name": "PetalBot",
   "operator": "Huawei",
   "category": "search",
   "robots_token": "PetalBot",
   "html": "https://www.pathwren.workers.dev/crawler/petalbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/petalbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/petalbot.md"
  },
  {
   "slug": "phindbot",
   "name": "PhindBot",
   "operator": "Phind",
   "category": "ai-search",
   "robots_token": "PhindBot",
   "html": "https://www.pathwren.workers.dev/crawler/phindbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/phindbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/phindbot.md"
  },
  {
   "slug": "pinterestbot",
   "name": "Pinterestbot",
   "operator": "Pinterest",
   "category": "search",
   "robots_token": "Pinterestbot",
   "html": "https://www.pathwren.workers.dev/crawler/pinterestbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/pinterestbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/pinterestbot.md"
  },
  {
   "slug": "poseidon-research-crawler",
   "name": "Poseidon Research Crawler",
   "operator": "Poseidon Research",
   "category": "ai-training",
   "robots_token": "Poseidon Research Crawler",
   "html": "https://www.pathwren.workers.dev/crawler/poseidon-research-crawler.html",
   "json": "https://www.pathwren.workers.dev/crawler/poseidon-research-crawler.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/poseidon-research-crawler.md"
  },
  {
   "slug": "qualifiedbot",
   "name": "QualifiedBot",
   "operator": "Qualified",
   "category": "ai-search",
   "robots_token": "QualifiedBot",
   "html": "https://www.pathwren.workers.dev/crawler/qualifiedbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/qualifiedbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/qualifiedbot.md"
  },
  {
   "slug": "quillbot",
   "name": "QuillBot",
   "operator": "QuillBot",
   "category": "ai-training",
   "robots_token": "QuillBot",
   "html": "https://www.pathwren.workers.dev/crawler/quillbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/quillbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/quillbot.md"
  },
  {
   "slug": "qwantbot",
   "name": "Qwantbot",
   "operator": "Qwant",
   "category": "search",
   "robots_token": "Qwantbot",
   "html": "https://www.pathwren.workers.dev/crawler/qwantbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/qwantbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/qwantbot.md"
  },
  {
   "slug": "qwantbot-news",
   "name": "Qwantbot-news",
   "operator": "Qwant",
   "category": "search",
   "robots_token": "Qwantbot-news",
   "html": "https://www.pathwren.workers.dev/crawler/qwantbot-news.html",
   "json": "https://www.pathwren.workers.dev/crawler/qwantbot-news.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/qwantbot-news.md"
  },
  {
   "slug": "reflectionbot",
   "name": "Reflectionbot",
   "operator": "Reflection AI",
   "category": "ai-training",
   "robots_token": "Reflectionbot",
   "html": "https://www.pathwren.workers.dev/crawler/reflectionbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/reflectionbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/reflectionbot.md"
  },
  {
   "slug": "rogerbot",
   "name": "rogerbot",
   "operator": "Moz",
   "category": "seo",
   "robots_token": "rogerbot",
   "html": "https://www.pathwren.workers.dev/crawler/rogerbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/rogerbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/rogerbot.md"
  },
  {
   "slug": "sbintuitionsbot",
   "name": "SBIntuitionsBot",
   "operator": "SB Intuitions",
   "category": "ai-training",
   "robots_token": "SBIntuitionsBot",
   "html": "https://www.pathwren.workers.dev/crawler/sbintuitionsbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/sbintuitionsbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/sbintuitionsbot.md"
  },
  {
   "slug": "scrapy",
   "name": "Scrapy",
   "operator": "Scrapy project",
   "category": "tool",
   "robots_token": "Scrapy",
   "html": "https://www.pathwren.workers.dev/crawler/scrapy.html",
   "json": "https://www.pathwren.workers.dev/crawler/scrapy.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/scrapy.md"
  },
  {
   "slug": "screaming-frog-seo-spider",
   "name": "Screaming Frog SEO Spider",
   "operator": "Screaming Frog",
   "category": "tool",
   "robots_token": "Screaming Frog SEO Spider",
   "html": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.html",
   "json": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/screaming-frog-seo-spider.md"
  },
  {
   "slug": "semrushbot",
   "name": "SemrushBot",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SemrushBot",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot.md"
  },
  {
   "slug": "semrushbot-ba",
   "name": "SemrushBot-BA",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SemrushBot-BA",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot-ba.md"
  },
  {
   "slug": "semrushbot-esi",
   "name": "SemrushBot-ESI",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SemrushBot-ESI",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot-esi.md"
  },
  {
   "slug": "semrushbot-ft",
   "name": "SemrushBot-FT",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SemrushBot-FT",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot-ft.md"
  },
  {
   "slug": "semrushbot-ocob",
   "name": "SemrushBot-OCOB",
   "operator": "Semrush",
   "category": "ai-training",
   "robots_token": "SemrushBot-OCOB",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot-ocob.md"
  },
  {
   "slug": "semrushbot-si",
   "name": "SemrushBot-SI",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SemrushBot-SI",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot-si.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot-si.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot-si.md"
  },
  {
   "slug": "semrushbot-swa",
   "name": "SemrushBot-SWA",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SemrushBot-SWA",
   "html": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.html",
   "json": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/semrushbot-swa.md"
  },
  {
   "slug": "seokicks",
   "name": "SEOkicks",
   "operator": "SEOkicks",
   "category": "seo",
   "robots_token": "SEOkicks",
   "html": "https://www.pathwren.workers.dev/crawler/seokicks.html",
   "json": "https://www.pathwren.workers.dev/crawler/seokicks.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/seokicks.md"
  },
  {
   "slug": "serpstatbot",
   "name": "serpstatbot",
   "operator": "Serpstat",
   "category": "seo",
   "robots_token": "serpstatbot",
   "html": "https://www.pathwren.workers.dev/crawler/serpstatbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/serpstatbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/serpstatbot.md"
  },
  {
   "slug": "seznambot",
   "name": "SeznamBot",
   "operator": "Seznam",
   "category": "search",
   "robots_token": "SeznamBot",
   "html": "https://www.pathwren.workers.dev/crawler/seznambot.html",
   "json": "https://www.pathwren.workers.dev/crawler/seznambot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/seznambot.md"
  },
  {
   "slug": "shapbot",
   "name": "ShapBot",
   "operator": "Parallel",
   "category": "ai-search",
   "robots_token": "ShapBot",
   "html": "https://www.pathwren.workers.dev/crawler/shapbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/shapbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/shapbot.md"
  },
  {
   "slug": "sidetrade-indexer-bot",
   "name": "Sidetrade indexer bot",
   "operator": "Sidetrade",
   "category": "ai-training",
   "robots_token": "Sidetrade indexer bot",
   "html": "https://www.pathwren.workers.dev/crawler/sidetrade-indexer-bot.html",
   "json": "https://www.pathwren.workers.dev/crawler/sidetrade-indexer-bot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/sidetrade-indexer-bot.md"
  },
  {
   "slug": "siteauditbot",
   "name": "SiteAuditBot",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SiteAuditBot",
   "html": "https://www.pathwren.workers.dev/crawler/siteauditbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/siteauditbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/siteauditbot.md"
  },
  {
   "slug": "slackbot",
   "name": "Slackbot",
   "operator": "Slack",
   "category": "preview",
   "robots_token": "Slackbot",
   "html": "https://www.pathwren.workers.dev/crawler/slackbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/slackbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/slackbot.md"
  },
  {
   "slug": "slackbot-linkexpanding",
   "name": "Slackbot-LinkExpanding",
   "operator": "Slack",
   "category": "preview",
   "robots_token": "Slackbot-LinkExpanding",
   "html": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.html",
   "json": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/slackbot-linkexpanding.md"
  },
  {
   "slug": "splitsignalbot",
   "name": "SplitSignalBot",
   "operator": "Semrush",
   "category": "seo",
   "robots_token": "SplitSignalBot",
   "html": "https://www.pathwren.workers.dev/crawler/splitsignalbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/splitsignalbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/splitsignalbot.md"
  },
  {
   "slug": "storebot-google",
   "name": "Storebot-Google",
   "operator": "Google",
   "category": "search",
   "robots_token": "Storebot-Google",
   "html": "https://www.pathwren.workers.dev/crawler/storebot-google.html",
   "json": "https://www.pathwren.workers.dev/crawler/storebot-google.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/storebot-google.md"
  },
  {
   "slug": "terracotta",
   "name": "TerraCotta",
   "operator": "Ceramic AI",
   "category": "ai-search",
   "robots_token": "TerraCotta",
   "html": "https://www.pathwren.workers.dev/crawler/terracotta.html",
   "json": "https://www.pathwren.workers.dev/crawler/terracotta.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/terracotta.md"
  },
  {
   "slug": "thinkbot",
   "name": "Thinkbot",
   "operator": "Thinkbot",
   "category": "dataset",
   "robots_token": "Thinkbot",
   "html": "https://www.pathwren.workers.dev/crawler/thinkbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/thinkbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/thinkbot.md"
  },
  {
   "slug": "tiktokspider",
   "name": "TikTokSpider",
   "operator": "ByteDance",
   "category": "ai-training",
   "robots_token": "TikTokSpider",
   "html": "https://www.pathwren.workers.dev/crawler/tiktokspider.html",
   "json": "https://www.pathwren.workers.dev/crawler/tiktokspider.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/tiktokspider.md"
  },
  {
   "slug": "timpibot",
   "name": "Timpibot",
   "operator": "Timpi",
   "category": "search",
   "robots_token": "Timpibot",
   "html": "https://www.pathwren.workers.dev/crawler/timpibot.html",
   "json": "https://www.pathwren.workers.dev/crawler/timpibot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/timpibot.md"
  },
  {
   "slug": "velenpublicwebcrawler",
   "name": "VelenPublicWebCrawler",
   "operator": "Hunter (Velen)",
   "category": "dataset",
   "robots_token": "VelenPublicWebCrawler",
   "html": "https://www.pathwren.workers.dev/crawler/velenpublicwebcrawler.html",
   "json": "https://www.pathwren.workers.dev/crawler/velenpublicwebcrawler.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/velenpublicwebcrawler.md"
  },
  {
   "slug": "webzio-extended",
   "name": "Webzio-Extended",
   "operator": "Webz.io",
   "category": "ai-training",
   "robots_token": "Webzio-Extended",
   "html": "https://www.pathwren.workers.dev/crawler/webzio-extended.html",
   "json": "https://www.pathwren.workers.dev/crawler/webzio-extended.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/webzio-extended.md"
  },
  {
   "slug": "wpbot",
   "name": "wpbot",
   "operator": "QuantumCloud",
   "category": "tool",
   "robots_token": "wpbot",
   "html": "https://www.pathwren.workers.dev/crawler/wpbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/wpbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/wpbot.md"
  },
  {
   "slug": "yak",
   "name": "YaK",
   "operator": "Meltwater",
   "category": "dataset",
   "robots_token": "YaK",
   "html": "https://www.pathwren.workers.dev/crawler/yak.html",
   "json": "https://www.pathwren.workers.dev/crawler/yak.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yak.md"
  },
  {
   "slug": "yandexadditional",
   "name": "YandexAdditional",
   "operator": "Yandex",
   "category": "ai-training",
   "robots_token": "YandexAdditional",
   "html": "https://www.pathwren.workers.dev/crawler/yandexadditional.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexadditional.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexadditional.md"
  },
  {
   "slug": "yandexadditionalbot",
   "name": "YandexAdditionalBot",
   "operator": "Yandex",
   "category": "ai-training",
   "robots_token": "YandexAdditionalBot",
   "html": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.md"
  },
  {
   "slug": "yandexblogs",
   "name": "YandexBlogs",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexBlogs",
   "html": "https://www.pathwren.workers.dev/crawler/yandexblogs.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexblogs.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexblogs.md"
  },
  {
   "slug": "yandexbot",
   "name": "YandexBot",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexBot",
   "html": "https://www.pathwren.workers.dev/crawler/yandexbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexbot.md"
  },
  {
   "slug": "yandexcalendar",
   "name": "YandexCalendar",
   "operator": "Yandex",
   "category": "user-fetch",
   "robots_token": "YandexCalendar",
   "html": "https://www.pathwren.workers.dev/crawler/yandexcalendar.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexcalendar.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexcalendar.md"
  },
  {
   "slug": "yandexcombot",
   "name": "YandexComBot",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexComBot",
   "html": "https://www.pathwren.workers.dev/crawler/yandexcombot.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexcombot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexcombot.md"
  },
  {
   "slug": "yandexdirect",
   "name": "YandexDirect",
   "operator": "Yandex",
   "category": "tool",
   "robots_token": "YandexDirect",
   "html": "https://www.pathwren.workers.dev/crawler/yandexdirect.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexdirect.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexdirect.md"
  },
  {
   "slug": "yandexfavicons",
   "name": "YandexFavicons",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexFavicons",
   "html": "https://www.pathwren.workers.dev/crawler/yandexfavicons.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexfavicons.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexfavicons.md"
  },
  {
   "slug": "yandeximages",
   "name": "YandexImages",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexImages",
   "html": "https://www.pathwren.workers.dev/crawler/yandeximages.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandeximages.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandeximages.md"
  },
  {
   "slug": "yandexmarket",
   "name": "YandexMarket",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexMarket",
   "html": "https://www.pathwren.workers.dev/crawler/yandexmarket.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexmarket.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexmarket.md"
  },
  {
   "slug": "yandexmedia",
   "name": "YandexMedia",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexMedia",
   "html": "https://www.pathwren.workers.dev/crawler/yandexmedia.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexmedia.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexmedia.md"
  },
  {
   "slug": "yandexmetrika",
   "name": "YandexMetrika",
   "operator": "Yandex",
   "category": "tool",
   "robots_token": "YandexMetrika",
   "html": "https://www.pathwren.workers.dev/crawler/yandexmetrika.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexmetrika.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexmetrika.md"
  },
  {
   "slug": "yandexmobilebot",
   "name": "YandexMobileBot",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexMobileBot",
   "html": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.md"
  },
  {
   "slug": "yandexrenderresourcesbot",
   "name": "YandexRenderResourcesBot",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexRenderResourcesBot",
   "html": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.md"
  },
  {
   "slug": "yandexscreenshotbot",
   "name": "YandexScreenshotBot",
   "operator": "Yandex",
   "category": "tool",
   "robots_token": "YandexScreenshotBot",
   "html": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.md"
  },
  {
   "slug": "yandexvideo",
   "name": "YandexVideo",
   "operator": "Yandex",
   "category": "search",
   "robots_token": "YandexVideo",
   "html": "https://www.pathwren.workers.dev/crawler/yandexvideo.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexvideo.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexvideo.md"
  },
  {
   "slug": "yandexwebmaster",
   "name": "YandexWebmaster",
   "operator": "Yandex",
   "category": "tool",
   "robots_token": "YandexWebmaster",
   "html": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.html",
   "json": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.md"
  },
  {
   "slug": "yeti",
   "name": "Yeti",
   "operator": "Naver",
   "category": "search",
   "robots_token": "Yeti",
   "html": "https://www.pathwren.workers.dev/crawler/yeti.html",
   "json": "https://www.pathwren.workers.dev/crawler/yeti.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/yeti.md"
  },
  {
   "slug": "youbot",
   "name": "YouBot",
   "operator": "You.com",
   "category": "ai-search",
   "robots_token": "YouBot",
   "html": "https://www.pathwren.workers.dev/crawler/youbot.html",
   "json": "https://www.pathwren.workers.dev/crawler/youbot.json",
   "markdown": "https://www.pathwren.workers.dev/crawler/youbot.md"
  }
 ],
 "generated_at": "2026-09-03T06:29:54+00:00",
 "license": {
  "data": "CC0-1.0",
  "url": "https://creativecommons.org/publicdomain/zero/1.0/"
 },
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/crawler/index.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/crawler/index.md",
   "type": "text/markdown"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/documents.json",
   "type": "application/json",
   "title": "Every document here with its ETag and last-modified date"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ]
}