{ "operator": "Yandex", "slug": "yandex", "docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html", "crawler_count": 17, "crawlers": [ { "slug": "yandexadditional", "name": "YandexAdditional", "operator": "Yandex", "operator_slug": "yandex", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "YandexAdditional", "user_agent_substring": "YandexAdditional", "user_agent_example": "Mozilla/5.0 (compatible; YandexAdditional/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The token that controls whether already-indexed pages may appear in Search with Yandex AI answers. Yandex's table says it makes no indexing requests of its own — it exists so a site can opt out of the generative answer without leaving the index.", "cost_of_blocking": "You disappear from Yandex's AI answers while staying in Yandex Search. This is Yandex's equivalent of Google-Extended, and it is the cheap opt-out most people are looking for.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexadditional.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexadditional.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexadditionalbot", "name": "YandexAdditionalBot", "operator": "Yandex", "operator_slug": "yandex", "category": "ai-training", "category_label": "AI training crawlers", "robots_token": "YandexAdditionalBot", "user_agent_substring": "YandexAdditionalBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexAdditionalBot/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The second token Yandex publishes for the same AI-answers opt-out. Both names appear in Yandex's own robot list, so a robots.txt that names only one of them is half a policy.", "cost_of_blocking": "Same as YandexAdditional: out of Yandex's AI answers, still in Yandex Search. Name both tokens or neither.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexadditionalbot.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexblogs", "name": "YandexBlogs", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexBlogs", "user_agent_substring": "YandexBlogs", "user_agent_example": "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Yandex's blog-search robot; it indexes post comments as well as posts.", "cost_of_blocking": "Comment threads and blog posts stop being findable through Yandex blog search.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexblogs.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexblogs.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexbot", "name": "YandexBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexBot", "user_agent_substring": "YandexBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Yandex's search crawler, which also feeds Alice and Yandex's generative answers.", "cost_of_blocking": "Removal from Yandex Search. Verify with reverse DNS to a yandex.ru, yandex.net or yandex.com host — YandexBot is among the most-spoofed user-agents there is.", "operator_docs": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html", "html_url": "https://www.pathwren.workers.dev/crawler/yandexbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexbot.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexcalendar", "name": "YandexCalendar", "operator": "Yandex", "operator_slug": "yandex", "category": "user-fetch", "category_label": "User-triggered fetchers", "robots_token": "YandexCalendar", "user_agent_substring": "YandexCalendar", "user_agent_example": "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Downloads calendar files a user subscribed to. Yandex notes these files are often in directories that are disallowed for indexing, which is why the general rules are not applied.", "cost_of_blocking": "Users who subscribed to a calendar you publish stop receiving updates.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexcalendar.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexcalendar.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexcombot", "name": "YandexComBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexComBot", "user_agent_substring": "YandexComBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexComBot/3.0; +http://ya.cc/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes content for Yandex search in languages other than Russian. Yandex documents that it can index content when there is no explicit robot-specific restriction — a * group is not one.", "cost_of_blocking": "You leave Yandex's non-Russian index. A rule naming this token is the only one that works on it.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexcombot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexcombot.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexdirect", "name": "YandexDirect", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexDirect", "user_agent_substring": "YandexDirect", "user_agent_example": "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Reads the content of Yandex Advertising Network partner pages to work out their topic so relevant ads can be matched. Documented as not taking the general robots.txt rules into account.", "cost_of_blocking": "Ads on your pages become less relevant and earn less. Relevant only if you monetise with Yandex's network.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexdirect.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexdirect.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexfavicons", "name": "YandexFavicons", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexFavicons", "user_agent_substring": "YandexFavicons", "user_agent_example": "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Downloads your favicon so Yandex can show it beside your result. Documented as not taking the general robots.txt rules into account.", "cost_of_blocking": "Your results in Yandex lose their icon. Cosmetic, and a * rule will not achieve it anyway.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexfavicons.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexfavicons.json", "last_reviewed": "2026-09-03" }, { "slug": "yandeximages", "name": "YandexImages", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexImages", "user_agent_substring": "YandexImages", "user_agent_example": "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes images for Yandex Images. Yandex's robot table marks it as taking the general robots.txt rules into account.", "cost_of_blocking": "Your images leave Yandex Images, which is a large share of image search in Russian-speaking markets.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandeximages.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandeximages.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexmarket", "name": "YandexMarket", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexMarket", "user_agent_substring": "YandexMarket", "user_agent_example": "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The robot behind Yandex Market, Yandex's shopping comparison service. Version 1.0 is documented as obeying the general rules; version 2.0 is documented as not.", "cost_of_blocking": "Your products stop being listed and priced in Yandex Market. For a retailer in that market this is a revenue block, not a bandwidth one.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmarket.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmarket.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexmedia", "name": "YandexMedia", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexMedia", "user_agent_substring": "YandexMedia", "user_agent_example": "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes multimedia data for Yandex. Takes the general robots.txt rules into account.", "cost_of_blocking": "Your multimedia content stops appearing in Yandex's media surfaces.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmedia.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmedia.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexmetrika", "name": "YandexMetrika", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexMetrika", "user_agent_substring": "YandexMetrika", "user_agent_example": "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", "respects_robots_txt": "by-design-no", "respects_robots_txt_label": "not governed by robots.txt (user-initiated, by operator policy)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Yandex Metrica's own fetcher. Two of its versions — the 2.0 yabs01 availability checker and the 4.0 CSS cache for Webvisor — are documented in Yandex's table as not using robots.txt at all.", "cost_of_blocking": "Nothing you can enforce through robots.txt. If you run Metrica, its session replay loses your stylesheets and renders your pages wrong.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmetrika.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmetrika.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexmobilebot", "name": "YandexMobileBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexMobileBot", "user_agent_substring": "YandexMobileBot", "user_agent_example": "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Decides whether a page's layout is suitable for mobile devices. Yandex's table marks it as NOT taking the general robots.txt rules into account, so a * group does not stop it — a group named YandexMobileBot does.", "cost_of_blocking": "Yandex loses its mobile-friendliness signal for your pages, which affects how they are ranked and rendered on phones.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexmobilebot.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexrenderresourcesbot", "name": "YandexRenderResourcesBot", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexRenderResourcesBot", "user_agent_substring": "YandexRenderResourcesBot", "user_agent_example": "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Loads the CSS, JavaScript and images Yandex needs to render a page. Yandex documents the exact rule: it ignores robots.txt for a resource when the HTML page using it is allowed, and does not fetch the resource when that page is disallowed.", "cost_of_blocking": "Yandex renders your pages without their stylesheets or scripts and ranks what it sees. This is the classic accidental self-inflicted ranking loss.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexrenderresourcesbot.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexscreenshotbot", "name": "YandexScreenshotBot", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexScreenshotBot", "user_agent_substring": "YandexScreenshotBot", "user_agent_example": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)", "respects_robots_txt": "own-token-only", "respects_robots_txt_label": "ignores the * group; obeys rules named for its own token", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Takes a screenshot of a page. Documented as not taking the general robots.txt rules into account.", "cost_of_blocking": "Yandex surfaces that show a page thumbnail show nothing for you.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexscreenshotbot.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexvideo", "name": "YandexVideo", "operator": "Yandex", "operator_slug": "yandex", "category": "search", "category_label": "Search engines", "robots_token": "YandexVideo", "user_agent_substring": "YandexVideo", "user_agent_example": "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "Indexes video for Yandex video search. Obeys the general robots.txt rules per Yandex's own table.", "cost_of_blocking": "Removal from Yandex video search. Note that a second robot, YandexVideoParser, does the same job and is documented as NOT taking the general rules into account.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexvideo.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexvideo.json", "last_reviewed": "2026-09-03" }, { "slug": "yandexwebmaster", "name": "YandexWebmaster", "operator": "Yandex", "operator_slug": "yandex", "category": "tool", "category_label": "Tools and frameworks", "robots_token": "YandexWebmaster", "user_agent_substring": "YandexWebmaster", "user_agent_example": "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", "respects_robots_txt": "documented", "respects_robots_txt_label": "obeys robots.txt (documented)", "verification_method": "reverse-dns", "verification_label": "reverse DNS", "published_ip_ranges_url": null, "ip_ranges_endpoint": null, "ipv4_prefix_count": 0, "ipv6_prefix_count": 0, "what_it_is": "The fetcher behind Yandex Webmaster, the console a site owner uses to inspect their own site.", "cost_of_blocking": "Your own Yandex Webmaster checks stop working. Blocking this only hurts you.", "operator_docs": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "html_url": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.html", "json_url": "https://www.pathwren.workers.dev/crawler/yandexwebmaster.json", "last_reviewed": "2026-09-03" } ], "ip_range_endpoints": [] }