{
 "name": "All 74 operators",
 "description": "Every company and project that runs a crawler in this index, with the crawlers it runs and a link to its own published documentation.",
 "url": "https://www.pathwren.workers.dev/operator",
 "canonical": "https://www.pathwren.workers.dev/operator/index.html",
 "formats": {
  "html": "https://www.pathwren.workers.dev/operator/index.html",
  "json": "https://www.pathwren.workers.dev/operator/index.json",
  "markdown": "https://www.pathwren.workers.dev/operator/index.md"
 },
 "curl": "curl -s https://www.pathwren.workers.dev/operator/openai.json",
 "count": 74,
 "entries": [
  {
   "slug": "ahrefs",
   "name": "Ahrefs",
   "crawlers": 2,
   "robots_tokens": [
    "AhrefsBot",
    "AhrefsSiteAudit"
   ],
   "operator_docs": "https://ahrefs.com/robot",
   "html": "https://www.pathwren.workers.dev/operator/ahrefs.html",
   "json": "https://www.pathwren.workers.dev/operator/ahrefs.json"
  },
  {
   "slug": "aihit",
   "name": "aiHit",
   "crawlers": 1,
   "robots_tokens": [
    "aiHitBot"
   ],
   "operator_docs": "https://www.aihitdata.com/about",
   "html": "https://www.pathwren.workers.dev/operator/aihit.html",
   "json": "https://www.pathwren.workers.dev/operator/aihit.json"
  },
  {
   "slug": "ai2",
   "name": "Allen Institute for AI",
   "crawlers": 2,
   "robots_tokens": [
    "AI2Bot",
    "Ai2Bot-Dolma"
   ],
   "operator_docs": "https://allenai.org/crawler",
   "html": "https://www.pathwren.workers.dev/operator/ai2.html",
   "json": "https://www.pathwren.workers.dev/operator/ai2.json"
  },
  {
   "slug": "amazon",
   "name": "Amazon",
   "crawlers": 2,
   "robots_tokens": [
    "Amazonbot",
    "bedrockbot"
   ],
   "operator_docs": "https://developer.amazon.com/amazonbot",
   "html": "https://www.pathwren.workers.dev/operator/amazon.html",
   "json": "https://www.pathwren.workers.dev/operator/amazon.json"
  },
  {
   "slug": "andi",
   "name": "Andi",
   "crawlers": 1,
   "robots_tokens": [
    "Andibot"
   ],
   "operator_docs": "https://andisearch.com/",
   "html": "https://www.pathwren.workers.dev/operator/andi.html",
   "json": "https://www.pathwren.workers.dev/operator/andi.json"
  },
  {
   "slug": "anthropic",
   "name": "Anthropic",
   "crawlers": 5,
   "robots_tokens": [
    "Claude-SearchBot",
    "Claude-User",
    "Claude-Web",
    "ClaudeBot",
    "anthropic-ai"
   ],
   "operator_docs": "https://support.anthropic.com/en/articles/8896518",
   "html": "https://www.pathwren.workers.dev/operator/anthropic.html",
   "json": "https://www.pathwren.workers.dev/operator/anthropic.json"
  },
  {
   "slug": "apple",
   "name": "Apple",
   "crawlers": 2,
   "robots_tokens": [
    "Applebot",
    "Applebot-Extended"
   ],
   "operator_docs": "https://support.apple.com/en-us/119829",
   "html": "https://www.pathwren.workers.dev/operator/apple.html",
   "json": "https://www.pathwren.workers.dev/operator/apple.json"
  },
  {
   "slug": "atlassian",
   "name": "Atlassian",
   "crawlers": 1,
   "robots_tokens": [
    "atlassian-bot"
   ],
   "operator_docs": "https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/",
   "html": "https://www.pathwren.workers.dev/operator/atlassian.html",
   "json": "https://www.pathwren.workers.dev/operator/atlassian.json"
  },
  {
   "slug": "awario",
   "name": "Awario",
   "crawlers": 2,
   "robots_tokens": [
    "AwarioRssBot",
    "AwarioSmartBot"
   ],
   "operator_docs": "https://awario.com/bots.html",
   "html": "https://www.pathwren.workers.dev/operator/awario.html",
   "json": "https://www.pathwren.workers.dev/operator/awario.json"
  },
  {
   "slug": "babbar",
   "name": "Babbar",
   "crawlers": 1,
   "robots_tokens": [
    "barkrowler"
   ],
   "operator_docs": "https://babbar.tech/crawler",
   "html": "https://www.pathwren.workers.dev/operator/babbar.html",
   "json": "https://www.pathwren.workers.dev/operator/babbar.json"
  },
  {
   "slug": "baidu",
   "name": "Baidu",
   "crawlers": 1,
   "robots_tokens": [
    "Baiduspider"
   ],
   "operator_docs": "https://help.baidu.com/question?prod_id=99&class=0&id=3001",
   "html": "https://www.pathwren.workers.dev/operator/baidu.html",
   "json": "https://www.pathwren.workers.dev/operator/baidu.json"
  },
  {
   "slug": "bytedance",
   "name": "ByteDance",
   "crawlers": 2,
   "robots_tokens": [
    "Bytespider",
    "TikTokSpider"
   ],
   "operator_docs": "https://www.bytespider.net/",
   "html": "https://www.pathwren.workers.dev/operator/bytedance.html",
   "json": "https://www.pathwren.workers.dev/operator/bytedance.json"
  },
  {
   "slug": "ceramic",
   "name": "Ceramic AI",
   "crawlers": 1,
   "robots_tokens": [
    "TerraCotta"
   ],
   "operator_docs": "https://ceramic.ai/",
   "html": "https://www.pathwren.workers.dev/operator/ceramic.html",
   "json": "https://www.pathwren.workers.dev/operator/ceramic.json"
  },
  {
   "slug": "cloudflare",
   "name": "Cloudflare",
   "crawlers": 1,
   "robots_tokens": [
    "Cloudflare-AutoRAG"
   ],
   "operator_docs": "https://developers.cloudflare.com/ai-search/",
   "html": "https://www.pathwren.workers.dev/operator/cloudflare.html",
   "json": "https://www.pathwren.workers.dev/operator/cloudflare.json"
  },
  {
   "slug": "cohere",
   "name": "Cohere",
   "crawlers": 2,
   "robots_tokens": [
    "cohere-ai",
    "cohere-training-data-crawler"
   ],
   "operator_docs": "https://cohere.com/",
   "html": "https://www.pathwren.workers.dev/operator/cohere.html",
   "json": "https://www.pathwren.workers.dev/operator/cohere.json"
  },
  {
   "slug": "commoncrawl",
   "name": "Common Crawl",
   "crawlers": 1,
   "robots_tokens": [
    "CCBot"
   ],
   "operator_docs": "https://commoncrawl.org/faq",
   "html": "https://www.pathwren.workers.dev/operator/commoncrawl.html",
   "json": "https://www.pathwren.workers.dev/operator/commoncrawl.json"
  },
  {
   "slug": "crawl4ai",
   "name": "Crawl4AI project",
   "crawlers": 1,
   "robots_tokens": [
    "Crawl4AI"
   ],
   "operator_docs": "https://github.com/unclecode/crawl4ai",
   "html": "https://www.pathwren.workers.dev/operator/crawl4ai.html",
   "json": "https://www.pathwren.workers.dev/operator/crawl4ai.json"
  },
  {
   "slug": "crawlspace",
   "name": "Crawlspace",
   "crawlers": 1,
   "robots_tokens": [
    "Crawlspace"
   ],
   "operator_docs": "https://crawlspace.dev",
   "html": "https://www.pathwren.workers.dev/operator/crawlspace.html",
   "json": "https://www.pathwren.workers.dev/operator/crawlspace.json"
  },
  {
   "slug": "dataforseo",
   "name": "DataForSEO",
   "crawlers": 1,
   "robots_tokens": [
    "DataForSeoBot"
   ],
   "operator_docs": "https://dataforseo.com/dataforseo-bot",
   "html": "https://www.pathwren.workers.dev/operator/dataforseo.html",
   "json": "https://www.pathwren.workers.dev/operator/dataforseo.json"
  },
  {
   "slug": "diffbot",
   "name": "Diffbot",
   "crawlers": 1,
   "robots_tokens": [
    "Diffbot"
   ],
   "operator_docs": "https://docs.diffbot.com/",
   "html": "https://www.pathwren.workers.dev/operator/diffbot.html",
   "json": "https://www.pathwren.workers.dev/operator/diffbot.json"
  },
  {
   "slug": "direqt",
   "name": "Direqt",
   "crawlers": 1,
   "robots_tokens": [
    "Anomura"
   ],
   "operator_docs": "https://direqt.ai",
   "html": "https://www.pathwren.workers.dev/operator/direqt.html",
   "json": "https://www.pathwren.workers.dev/operator/direqt.json"
  },
  {
   "slug": "duckduckgo",
   "name": "DuckDuckGo",
   "crawlers": 2,
   "robots_tokens": [
    "DuckAssistBot",
    "DuckDuckBot"
   ],
   "operator_docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/",
   "html": "https://www.pathwren.workers.dev/operator/duckduckgo.html",
   "json": "https://www.pathwren.workers.dev/operator/duckduckgo.json"
  },
  {
   "slug": "echobox",
   "name": "Echobox",
   "crawlers": 1,
   "robots_tokens": [
    "EchoboxBot"
   ],
   "operator_docs": "https://echobox.com",
   "html": "https://www.pathwren.workers.dev/operator/echobox.html",
   "json": "https://www.pathwren.workers.dev/operator/echobox.json"
  },
  {
   "slug": "exa",
   "name": "Exa",
   "crawlers": 1,
   "robots_tokens": [
    "ExaSearchBot"
   ],
   "operator_docs": "https://exa.ai",
   "html": "https://www.pathwren.workers.dev/operator/exa.html",
   "json": "https://www.pathwren.workers.dev/operator/exa.json"
  },
  {
   "slug": "factset",
   "name": "FactSet",
   "crawlers": 1,
   "robots_tokens": [
    "Factset_spyderbot"
   ],
   "operator_docs": "https://www.factset.com/ai",
   "html": "https://www.pathwren.workers.dev/operator/factset.html",
   "json": "https://www.pathwren.workers.dev/operator/factset.json"
  },
  {
   "slug": "firecrawl",
   "name": "Firecrawl",
   "crawlers": 1,
   "robots_tokens": [
    "FirecrawlAgent"
   ],
   "operator_docs": "https://docs.firecrawl.dev/",
   "html": "https://www.pathwren.workers.dev/operator/firecrawl.html",
   "json": "https://www.pathwren.workers.dev/operator/firecrawl.json"
  },
  {
   "slug": "google",
   "name": "Google",
   "crawlers": 26,
   "robots_tokens": [
    "APIs-Google",
    "AdsBot-Google",
    "AdsBot-Google-Mobile",
    "AdsBot-Google-Mobile-Apps",
    "FeedFetcher-Google",
    "Google-Agent",
    "Google-CWS",
    "Google-CloudVertexBot",
    "Google-Extended",
    "Google-GeminiNotebook",
    "Google-InspectionTool",
    "Google-Pinpoint",
    "Google-Read-Aloud",
    "Google-Safety",
    "Google-Site-Verification",
    "GoogleMessages",
    "GoogleOther",
    "GoogleOther-Image",
    "GoogleOther-Video",
    "GoogleProducer",
    "Googlebot",
    "Googlebot-Image",
    "Googlebot-News",
    "Googlebot-Video",
    "Mediapartners-Google",
    "Storebot-Google"
   ],
   "operator_docs": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
   "html": "https://www.pathwren.workers.dev/operator/google.html",
   "json": "https://www.pathwren.workers.dev/operator/google.json"
  },
  {
   "slug": "hive",
   "name": "Hive AI",
   "crawlers": 1,
   "robots_tokens": [
    "ImagesiftBot"
   ],
   "operator_docs": "https://imagesift.com/about",
   "html": "https://www.pathwren.workers.dev/operator/hive.html",
   "json": "https://www.pathwren.workers.dev/operator/hive.json"
  },
  {
   "slug": "huawei",
   "name": "Huawei",
   "crawlers": 1,
   "robots_tokens": [
    "PetalBot"
   ],
   "operator_docs": "https://aspiegel.com/petalbot",
   "html": "https://www.pathwren.workers.dev/operator/huawei.html",
   "json": "https://www.pathwren.workers.dev/operator/huawei.json"
  },
  {
   "slug": "hunter",
   "name": "Hunter (Velen)",
   "crawlers": 1,
   "robots_tokens": [
    "VelenPublicWebCrawler"
   ],
   "operator_docs": "https://velen.io/",
   "html": "https://www.pathwren.workers.dev/operator/hunter.html",
   "json": "https://www.pathwren.workers.dev/operator/hunter.json"
  },
  {
   "slug": "internetarchive",
   "name": "Internet Archive",
   "crawlers": 2,
   "robots_tokens": [
    "archive.org_bot",
    "ia_archiver"
   ],
   "operator_docs": "https://archive.org/details/archive.org_bot",
   "html": "https://www.pathwren.workers.dev/operator/internetarchive.html",
   "json": "https://www.pathwren.workers.dev/operator/internetarchive.json"
  },
  {
   "slug": "iss",
   "name": "ISS Corporate Solutions",
   "crawlers": 1,
   "robots_tokens": [
    "ISSCyberRiskCrawler"
   ],
   "operator_docs": "https://iss-cyber.com",
   "html": "https://www.pathwren.workers.dev/operator/iss.html",
   "json": "https://www.pathwren.workers.dev/operator/iss.json"
  },
  {
   "slug": "kagi",
   "name": "Kagi",
   "crawlers": 1,
   "robots_tokens": [
    "Kagibot"
   ],
   "operator_docs": "https://kagi.com/bot",
   "html": "https://www.pathwren.workers.dev/operator/kagi.html",
   "json": "https://www.pathwren.workers.dev/operator/kagi.json"
  },
  {
   "slug": "klaviyo",
   "name": "Klaviyo",
   "crawlers": 1,
   "robots_tokens": [
    "KlaviyoAIBot"
   ],
   "operator_docs": "https://help.klaviyo.com/hc/en-us/articles/40496146232219",
   "html": "https://www.pathwren.workers.dev/operator/klaviyo.html",
   "json": "https://www.pathwren.workers.dev/operator/klaviyo.json"
  },
  {
   "slug": "laion",
   "name": "LAION / img2dataset",
   "crawlers": 2,
   "robots_tokens": [
    "LAIONDownloader",
    "img2dataset"
   ],
   "operator_docs": "https://github.com/rom1504/img2dataset",
   "html": "https://www.pathwren.workers.dev/operator/laion.html",
   "json": "https://www.pathwren.workers.dev/operator/laion.json"
  },
  {
   "slug": "lightpanda",
   "name": "Lightpanda",
   "crawlers": 1,
   "robots_tokens": [
    "Lightpanda"
   ],
   "operator_docs": "https://lightpanda.io/",
   "html": "https://www.pathwren.workers.dev/operator/lightpanda.html",
   "json": "https://www.pathwren.workers.dev/operator/lightpanda.json"
  },
  {
   "slug": "linguee",
   "name": "Linguee",
   "crawlers": 1,
   "robots_tokens": [
    "Linguee Bot"
   ],
   "operator_docs": "https://www.linguee.com",
   "html": "https://www.pathwren.workers.dev/operator/linguee.html",
   "json": "https://www.pathwren.workers.dev/operator/linguee.json"
  },
  {
   "slug": "lyrenth",
   "name": "Lyrenth",
   "crawlers": 1,
   "robots_tokens": [
    "AIWebIndex"
   ],
   "operator_docs": "https://lyrenth.com/crawler-policy",
   "html": "https://www.pathwren.workers.dev/operator/lyrenth.html",
   "json": "https://www.pathwren.workers.dev/operator/lyrenth.json"
  },
  {
   "slug": "majestic",
   "name": "Majestic",
   "crawlers": 1,
   "robots_tokens": [
    "MJ12bot"
   ],
   "operator_docs": "https://mj12bot.com/",
   "html": "https://www.pathwren.workers.dev/operator/majestic.html",
   "json": "https://www.pathwren.workers.dev/operator/majestic.json"
  },
  {
   "slug": "meltwater",
   "name": "Meltwater",
   "crawlers": 1,
   "robots_tokens": [
    "YaK"
   ],
   "operator_docs": "https://www.meltwater.com/en/suite/consumer-intelligence",
   "html": "https://www.pathwren.workers.dev/operator/meltwater.html",
   "json": "https://www.pathwren.workers.dev/operator/meltwater.json"
  },
  {
   "slug": "meta",
   "name": "Meta",
   "crawlers": 5,
   "robots_tokens": [
    "FacebookBot",
    "Meta-WebIndexer",
    "facebookexternalhit",
    "meta-externalagent",
    "meta-externalfetcher"
   ],
   "operator_docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
   "html": "https://www.pathwren.workers.dev/operator/meta.html",
   "json": "https://www.pathwren.workers.dev/operator/meta.json"
  },
  {
   "slug": "microsoft",
   "name": "Microsoft",
   "crawlers": 1,
   "robots_tokens": [
    "bingbot"
   ],
   "operator_docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
   "html": "https://www.pathwren.workers.dev/operator/microsoft.html",
   "json": "https://www.pathwren.workers.dev/operator/microsoft.json"
  },
  {
   "slug": "mistral",
   "name": "Mistral AI",
   "crawlers": 1,
   "robots_tokens": [
    "MistralAI-User"
   ],
   "operator_docs": "https://docs.mistral.ai/",
   "html": "https://www.pathwren.workers.dev/operator/mistral.html",
   "json": "https://www.pathwren.workers.dev/operator/mistral.json"
  },
  {
   "slug": "mojeek",
   "name": "Mojeek",
   "crawlers": 1,
   "robots_tokens": [
    "MojeekBot"
   ],
   "operator_docs": "https://www.mojeek.com/bot.html",
   "html": "https://www.pathwren.workers.dev/operator/mojeek.html",
   "json": "https://www.pathwren.workers.dev/operator/mojeek.json"
  },
  {
   "slug": "moz",
   "name": "Moz",
   "crawlers": 2,
   "robots_tokens": [
    "dotbot",
    "rogerbot"
   ],
   "operator_docs": "https://moz.com/help/moz-procedures/crawlers/dotbot",
   "html": "https://www.pathwren.workers.dev/operator/moz.html",
   "json": "https://www.pathwren.workers.dev/operator/moz.json"
  },
  {
   "slug": "naver",
   "name": "Naver",
   "crawlers": 1,
   "robots_tokens": [
    "Yeti"
   ],
   "operator_docs": "https://searchadvisor.naver.com/guide/seo-basic-crawl",
   "html": "https://www.pathwren.workers.dev/operator/naver.html",
   "json": "https://www.pathwren.workers.dev/operator/naver.json"
  },
  {
   "slug": "nict",
   "name": "NICT",
   "crawlers": 1,
   "robots_tokens": [
    "ICC-Crawler"
   ],
   "operator_docs": "https://www.nict.go.jp/en/",
   "html": "https://www.pathwren.workers.dev/operator/nict.html",
   "json": "https://www.pathwren.workers.dev/operator/nict.json"
  },
  {
   "slug": "openai",
   "name": "OpenAI",
   "crawlers": 4,
   "robots_tokens": [
    "ChatGPT-User",
    "GPTBot",
    "OAI-SearchBot"
   ],
   "operator_docs": "https://platform.openai.com/docs/bots",
   "html": "https://www.pathwren.workers.dev/operator/openai.html",
   "json": "https://www.pathwren.workers.dev/operator/openai.json"
  },
  {
   "slug": "panscient",
   "name": "Panscient",
   "crawlers": 1,
   "robots_tokens": [
    "panscient.com"
   ],
   "operator_docs": "https://panscient.com/faq.htm",
   "html": "https://www.pathwren.workers.dev/operator/panscient.html",
   "json": "https://www.pathwren.workers.dev/operator/panscient.json"
  },
  {
   "slug": "parallel",
   "name": "Parallel",
   "crawlers": 1,
   "robots_tokens": [
    "ShapBot"
   ],
   "operator_docs": "https://docs.parallel.ai/features/crawler",
   "html": "https://www.pathwren.workers.dev/operator/parallel.html",
   "json": "https://www.pathwren.workers.dev/operator/parallel.json"
  },
  {
   "slug": "perplexity",
   "name": "Perplexity",
   "crawlers": 2,
   "robots_tokens": [
    "Perplexity-User",
    "PerplexityBot"
   ],
   "operator_docs": "https://docs.perplexity.ai/guides/bots",
   "html": "https://www.pathwren.workers.dev/operator/perplexity.html",
   "json": "https://www.pathwren.workers.dev/operator/perplexity.json"
  },
  {
   "slug": "phind",
   "name": "Phind",
   "crawlers": 1,
   "robots_tokens": [
    "PhindBot"
   ],
   "operator_docs": "https://www.phind.com/",
   "html": "https://www.pathwren.workers.dev/operator/phind.html",
   "json": "https://www.pathwren.workers.dev/operator/phind.json"
  },
  {
   "slug": "pinterest",
   "name": "Pinterest",
   "crawlers": 1,
   "robots_tokens": [
    "Pinterestbot"
   ],
   "operator_docs": "https://help.pinterest.com/en/business/article/pinterest-crawler",
   "html": "https://www.pathwren.workers.dev/operator/pinterest.html",
   "json": "https://www.pathwren.workers.dev/operator/pinterest.json"
  },
  {
   "slug": "poseidon",
   "name": "Poseidon Research",
   "crawlers": 1,
   "robots_tokens": [
    "Poseidon Research Crawler"
   ],
   "operator_docs": "https://www.poseidonresearch.com",
   "html": "https://www.pathwren.workers.dev/operator/poseidon.html",
   "json": "https://www.pathwren.workers.dev/operator/poseidon.json"
  },
  {
   "slug": "qualified",
   "name": "Qualified",
   "crawlers": 1,
   "robots_tokens": [
    "QualifiedBot"
   ],
   "operator_docs": "https://www.qualified.com",
   "html": "https://www.pathwren.workers.dev/operator/qualified.html",
   "json": "https://www.pathwren.workers.dev/operator/qualified.json"
  },
  {
   "slug": "quantumcloud",
   "name": "QuantumCloud",
   "crawlers": 1,
   "robots_tokens": [
    "wpbot"
   ],
   "operator_docs": "https://www.quantumcloud.com",
   "html": "https://www.pathwren.workers.dev/operator/quantumcloud.html",
   "json": "https://www.pathwren.workers.dev/operator/quantumcloud.json"
  },
  {
   "slug": "quillbot",
   "name": "QuillBot",
   "crawlers": 1,
   "robots_tokens": [
    "QuillBot"
   ],
   "operator_docs": "https://quillbot.com",
   "html": "https://www.pathwren.workers.dev/operator/quillbot.html",
   "json": "https://www.pathwren.workers.dev/operator/quillbot.json"
  },
  {
   "slug": "qwant",
   "name": "Qwant",
   "crawlers": 2,
   "robots_tokens": [
    "Qwantbot",
    "Qwantbot-news"
   ],
   "operator_docs": "https://help.qwant.com/bot/",
   "html": "https://www.pathwren.workers.dev/operator/qwant.html",
   "json": "https://www.pathwren.workers.dev/operator/qwant.json"
  },
  {
   "slug": "reflection",
   "name": "Reflection AI",
   "crawlers": 1,
   "robots_tokens": [
    "Reflectionbot"
   ],
   "operator_docs": "https://reflection.ai/",
   "html": "https://www.pathwren.workers.dev/operator/reflection.html",
   "json": "https://www.pathwren.workers.dev/operator/reflection.json"
  },
  {
   "slug": "rois",
   "name": "ROIS-DS",
   "crawlers": 1,
   "robots_tokens": [
    "Cotoyogi"
   ],
   "operator_docs": "https://ds.rois.ac.jp/en_center8/en_crawler/",
   "html": "https://www.pathwren.workers.dev/operator/rois.html",
   "json": "https://www.pathwren.workers.dev/operator/rois.json"
  },
  {
   "slug": "sbintuitions",
   "name": "SB Intuitions",
   "crawlers": 1,
   "robots_tokens": [
    "SBIntuitionsBot"
   ],
   "operator_docs": "https://www.sbintuitions.co.jp/en/bot/",
   "html": "https://www.pathwren.workers.dev/operator/sbintuitions.html",
   "json": "https://www.pathwren.workers.dev/operator/sbintuitions.json"
  },
  {
   "slug": "scrapy",
   "name": "Scrapy project",
   "crawlers": 1,
   "robots_tokens": [
    "Scrapy"
   ],
   "operator_docs": "https://scrapy.org/",
   "html": "https://www.pathwren.workers.dev/operator/scrapy.html",
   "json": "https://www.pathwren.workers.dev/operator/scrapy.json"
  },
  {
   "slug": "screamingfrog",
   "name": "Screaming Frog",
   "crawlers": 1,
   "robots_tokens": [
    "Screaming Frog SEO Spider"
   ],
   "operator_docs": "https://www.screamingfrog.co.uk/seo-spider/user-agent/",
   "html": "https://www.pathwren.workers.dev/operator/screamingfrog.html",
   "json": "https://www.pathwren.workers.dev/operator/screamingfrog.json"
  },
  {
   "slug": "semrush",
   "name": "Semrush",
   "crawlers": 9,
   "robots_tokens": [
    "SemrushBot",
    "SemrushBot-BA",
    "SemrushBot-ESI",
    "SemrushBot-FT",
    "SemrushBot-OCOB",
    "SemrushBot-SI",
    "SemrushBot-SWA",
    "SiteAuditBot",
    "SplitSignalBot"
   ],
   "operator_docs": "https://www.semrush.com/bot/",
   "html": "https://www.pathwren.workers.dev/operator/semrush.html",
   "json": "https://www.pathwren.workers.dev/operator/semrush.json"
  },
  {
   "slug": "seokicks",
   "name": "SEOkicks",
   "crawlers": 1,
   "robots_tokens": [
    "SEOkicks"
   ],
   "operator_docs": "https://www.seokicks.de/robot.html",
   "html": "https://www.pathwren.workers.dev/operator/seokicks.html",
   "json": "https://www.pathwren.workers.dev/operator/seokicks.json"
  },
  {
   "slug": "serpstat",
   "name": "Serpstat",
   "crawlers": 1,
   "robots_tokens": [
    "serpstatbot"
   ],
   "operator_docs": "https://serpstatbot.com/",
   "html": "https://www.pathwren.workers.dev/operator/serpstat.html",
   "json": "https://www.pathwren.workers.dev/operator/serpstat.json"
  },
  {
   "slug": "seznam",
   "name": "Seznam",
   "crawlers": 1,
   "robots_tokens": [
    "SeznamBot"
   ],
   "operator_docs": "https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/",
   "html": "https://www.pathwren.workers.dev/operator/seznam.html",
   "json": "https://www.pathwren.workers.dev/operator/seznam.json"
  },
  {
   "slug": "sidetrade",
   "name": "Sidetrade",
   "crawlers": 1,
   "robots_tokens": [
    "Sidetrade indexer bot"
   ],
   "operator_docs": "https://www.sidetrade.com",
   "html": "https://www.pathwren.workers.dev/operator/sidetrade.html",
   "json": "https://www.pathwren.workers.dev/operator/sidetrade.json"
  },
  {
   "slug": "slack",
   "name": "Slack",
   "crawlers": 2,
   "robots_tokens": [
    "Slackbot",
    "Slackbot-LinkExpanding"
   ],
   "operator_docs": "https://api.slack.com/robots",
   "html": "https://www.pathwren.workers.dev/operator/slack.html",
   "json": "https://www.pathwren.workers.dev/operator/slack.json"
  },
  {
   "slug": "thinkbot",
   "name": "Thinkbot",
   "crawlers": 1,
   "robots_tokens": [
    "Thinkbot"
   ],
   "operator_docs": "https://www.thinkbot.agency",
   "html": "https://www.pathwren.workers.dev/operator/thinkbot.html",
   "json": "https://www.pathwren.workers.dev/operator/thinkbot.json"
  },
  {
   "slug": "timpi",
   "name": "Timpi",
   "crawlers": 1,
   "robots_tokens": [
    "Timpibot"
   ],
   "operator_docs": "https://timpi.io/",
   "html": "https://www.pathwren.workers.dev/operator/timpi.html",
   "json": "https://www.pathwren.workers.dev/operator/timpi.json"
  },
  {
   "slug": "webz",
   "name": "Webz.io",
   "crawlers": 3,
   "robots_tokens": [
    "Webzio-Extended",
    "omgili",
    "omgilibot"
   ],
   "operator_docs": "https://webz.io/blog/machine-learning/",
   "html": "https://www.pathwren.workers.dev/operator/webz.html",
   "json": "https://www.pathwren.workers.dev/operator/webz.json"
  },
  {
   "slug": "yandex",
   "name": "Yandex",
   "crawlers": 17,
   "robots_tokens": [
    "YandexAdditional",
    "YandexAdditionalBot",
    "YandexBlogs",
    "YandexBot",
    "YandexCalendar",
    "YandexComBot",
    "YandexDirect",
    "YandexFavicons",
    "YandexImages",
    "YandexMarket",
    "YandexMedia",
    "YandexMetrika",
    "YandexMobileBot",
    "YandexRenderResourcesBot",
    "YandexScreenshotBot",
    "YandexVideo",
    "YandexWebmaster"
   ],
   "operator_docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html",
   "html": "https://www.pathwren.workers.dev/operator/yandex.html",
   "json": "https://www.pathwren.workers.dev/operator/yandex.json"
  },
  {
   "slug": "you",
   "name": "You.com",
   "crawlers": 1,
   "robots_tokens": [
    "YouBot"
   ],
   "operator_docs": "https://about.you.com/youbot/",
   "html": "https://www.pathwren.workers.dev/operator/you.html",
   "json": "https://www.pathwren.workers.dev/operator/you.json"
  }
 ],
 "generated_at": "2026-09-03T06:29:54+00:00",
 "license": {
  "data": "CC0-1.0",
  "url": "https://creativecommons.org/publicdomain/zero/1.0/"
 },
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/operator/index.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=111",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 111,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/operator/index.md",
   "type": "text/markdown"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/documents.json",
   "type": "application/json",
   "title": "Every document here with its ETag and last-modified date"
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  }
 ]
}