{ "generated": "2026-10-26T12:06:16+01:00", "source": "https://github.com/ai-robots-txt/ai.robots.txt", "license": "MIT (ai.robots.txt)", "fetched": "2026-10-26T12:06:16+01:00", "info": "https://ipv64.net/v64_blocklists", "count": 200, "count_user": 60, "match": "case-insensitive token in the User-Agent header; user = fetch on behalf of a user (ChatGPT-User, Perplexity-User, assistants, agents)", "bots": [ { "name": "AddSearchBot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search results. More info can be found at https://knownagents.com/agents/addsearchbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "AgentDataBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "AgentDataBot visits public web pages to identify technology stacks, company details, and team members. The data builds AgentData's company-intelligence database for AI ag… More info can be found at https://knownagents.com/agents/agentdatabot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "AgentTimes", "user": false, "operator": "[The Agent Times](https://theagenttimes.com/about)", "function": "Data Scraper from RSS Feeds.", "description": "Scrapes data for AI news aggregation and republishing.", "respect": "Unclear at this time.", "frequency": "Requests RSS feed every 5-6 minutes." }, { "name": "AGI Agent", "user": true, "operator": "AGI, Inc", "function": "AI Agents", "description": "AGI Agent is an AI assistant operated by AGI, Inc. that navigates websites to complete user-directed tasks, including online purchases and other actions. More info can be found at https://knownagents.com/agents/agi-agent", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "AI2Bot", "user": false, "operator": "[Ai2](https://allenai.org/crawler)", "function": "Content is used to train open language models.", "description": "Explores 'certain domains' to find web content.", "respect": "Yes", "frequency": "No information provided." }, { "name": "AI2Bot-DeepResearchEval", "user": true, "operator": "Ai2, a non-profit AI research institute", "function": "AI Assistants", "description": "Ai2Bot-DeepResearchEval is operated by Ai2, a non-profit AI research institute. It's used to collect and scan resources used in deep research queries performed by Ai2's o… More info can be found at https://knownagents.com/agents/ai2bot-deepresearcheval", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Ai2Bot-Dolma", "user": false, "operator": "[Ai2](https://allenai.org/crawler)", "function": "Content is used to train open language models.", "description": "Explores 'certain domains' to find web content.", "respect": "Yes", "frequency": "No information provided." }, { "name": "aiHitBot", "user": false, "operator": "[aiHit](https://www.aihitdata.com/about)", "function": "A massive, artificial intelligence/machine learning, automated system.", "description": "Scrapes data for AI systems.", "respect": "Yes", "frequency": "No information provided." }, { "name": "AIWebIndex", "user": false, "operator": "[Lyrenth](https://lyrenth.com)", "function": "AI Search Crawlers", "description": "Builds an index of public pages and serves them to AI agents as extracted, readable text with attribution and a link back to the source. Does not train foundation models on crawled content. Identity can be checked three ways: published IP ranges at https://lyrenth.com/bot/ip-ranges.json, forward-confirmed reverse DNS under lyrenth.com, and Web Bot Auth signatures (RFC 9421). Full policy at https://lyrenth.com/crawler-policy", "respect": "[Yes](https://lyrenth.com/crawler-policy)", "frequency": "At most one request per domain every 2 seconds, and slower where robots.txt sets a longer Crawl-delay." }, { "name": "amazon-kendra", "user": false, "operator": "Amazon", "function": "Collects data for AI natural language search", "description": "Amazon Kendra is a highly accurate intelligent search service that enables your users to search unstructured data using natural language. It returns specific answers to questions, giving users an experience that's close to interacting with a human expert. It is highly scalable and capable of meeting performance demands, tightly integrated with other AWS services such as Amazon S3 and Amazon Lex, and offers enterprise-grade security.", "respect": "Yes", "frequency": "No information provided." }, { "name": "amazon-QBusiness", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "amazon-QBusiness is an Amazon Q Business web crawler that fetches and indexes web content for Amazon Q Business applications. More info can be found at https://knownagents.com/agents/amazon-qbusiness", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Amazonbot", "user": false, "operator": "Amazon", "function": "Service improvement and enabling answers for Alexa users.", "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses.", "respect": "Yes", "frequency": "No information provided." }, { "name": "AmazonBuyForMe", "user": true, "operator": "[Amazon](https://amazon.com)", "function": "AI Agents", "description": "AmazonBuyForMe is an Amazon bot that crawls websites as part of the Amazon Buy For Me service. This bot visits product pages and e-commerce websites to gather product inf… More info can be found at https://knownagents.com/agents/amazonbuyforme", "respect": "Unclear at this time.", "frequency": "No information provided." }, { "name": "Amzn-SearchBot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/amzn-searchbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Amzn-User", "user": true, "operator": "Amazon, used for fetching web content to answer user queries through Alexa and other Amazon AI services", "function": "AI Assistants", "description": "Amzn-User is an AI assistant operated by Amazon, used for fetching web content to answer user queries through Alexa and other Amazon AI services. More info can be found at https://knownagents.com/agents/amzn-user", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Andibot", "user": false, "operator": "[Andi](https://andisearch.com/)", "function": "Search engine using generative AI, AI Search Assistant", "description": "Scrapes website and provides AI summary.", "respect": "Unclear at this time", "frequency": "No information provided." }, { "name": "Anomura", "user": false, "operator": "[Direqt](https://direqt.ai)", "function": "Collects data for AI search", "description": "Anomura is Direqt's search crawler, it discovers and indexes pages their customers websites.", "respect": "Yes", "frequency": "No information provided." }, { "name": "anthropic-ai", "user": false, "operator": "[Anthropic](https://www.anthropic.com)", "function": "Scrapes data to train Anthropic's AI products.", "description": "Scrapes data to train LLMs and AI products offered by Anthropic.", "respect": "Unclear at this time.", "frequency": "No information provided." }, { "name": "ApifyBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "ApifyBot is a web scraping and data extraction crawler by Apify that collects website content for use in AI, LLMs, RAG, and automation workflows. More info can be found at https://knownagents.com/agents/apifybot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ApifyWebsiteContentCrawler", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "ApifyWebsiteContentCrawler is a web crawler by Apify that extracts and downloads full website content for use in AI, data analysis, and automation workflows. More info can be found at https://knownagents.com/agents/apifywebsitecontentcrawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Applebot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://knownagents.com/agents/applebot", "respect": "[Yes](https://support.apple.com/en-us/119829#retrieval)", "frequency": "Unclear at this time." }, { "name": "Applebot-Extended", "user": false, "operator": "[Apple](https://support.apple.com/en-us/119829#datausage)", "function": "Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others.", "description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools.", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "Aranet-SearchBot", "user": false, "operator": "Unclear at this time.", "function": "Undocumented AI Agents", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/aranet-searchbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "atlassian-bot", "user": false, "operator": "[Atlassian](https://www.atlassian.com)", "function": "AI search, assistants and agents", "description": "atlassian-bot is a web crawler used to index website content for its AI search, assistants and agents available in its Rovo GenAI product.", "respect": "[Yes](https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/#Editing-your-robots.txt)", "frequency": "No information provided." }, { "name": "Awario", "user": false, "operator": "Awario", "function": "AI Data Scrapers", "description": "Awario is an AI data scraper operated by Awario. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/awario", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "AzureAI-SearchBot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/azureai-searchbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "bedrockbot", "user": false, "operator": "[Amazon](https://amazon.com)", "function": "Data scraping for custom AI applications.", "description": "Connects to and crawls URLs that have been selected for use in a user's AWS bedrock application.", "respect": "[Yes](https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html#configuration-webcrawl-connector)", "frequency": "Unclear at this time." }, { "name": "bigsur.ai", "user": true, "operator": "Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi…", "function": "AI Assistants", "description": "bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi… More info can be found at https://knownagents.com/agents/bigsur-ai", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "BixelBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "BixelBot collects public company and market signals for Bixel's structured company-data platform, which supplies verified business information to AI agents and developers… More info can be found at https://knownagents.com/agents/bixelbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Bravebot", "user": false, "operator": "https://safe.search.brave.com/help/brave-search-crawler", "function": "AI Data Providers", "description": "Bravebot is a web crawler by Brave that indexes pages for Brave Search, providing search data and AI-optimized context to power chatbots, agents, and RAG pipelines. More info can be found at https://knownagents.com/agents/bravebot", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "Brightbot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "Brightbot is a web data collection crawler by Bright Data that extracts and structures public website content at scale, providing AI-ready data for model training, RAG pi… More info can be found at https://knownagents.com/agents/brightbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Brightbot 1.0", "user": false, "operator": "https://brightdata.com/brightbot", "function": "LLM/AI training.", "description": "Scrapes data to train LLMs and AI products focused on website customer support, [uses residential IPs and legit-looking user-agents to disguise itself](https://ksol.io/en/blog/posts/brightbot-not-that-bright/).", "respect": "Unclear at this time.", "frequency": "At least one per minute." }, { "name": "BuddyBot", "user": false, "operator": "[BuddyBotLearning](https://www.buddybotlearning.com)", "function": "AI Learning Companion", "description": "BuddyBot is a voice-controlled AI learning companion targeted at childhooded STEM education.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Bytespider", "user": false, "operator": "ByteDance", "function": "LLM training.", "description": "Downloads data to train LLMS, including ChatGPT competitors.", "respect": "No", "frequency": "Unclear at this time." }, { "name": "CartAI Agentic Commerce", "user": true, "operator": "CartAI that accesses online stores to support user-directed shopping and checkout workflows", "function": "AI Agents", "description": "CartAI Agentic Commerce is a signed browser agent operated by CartAI that accesses online stores to support user-directed shopping and checkout workflows. More info can be found at https://knownagents.com/agents/cartai-agentic-commerce", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "CCBot", "user": false, "operator": "[Common Crawl Foundation](https://commoncrawl.org)", "function": "Provides open crawl dataset, used for many purposes, including Machine Learning/AI.", "description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers).", "respect": "[Yes](https://commoncrawl.org/ccbot)", "frequency": "Monthly at present." }, { "name": "Channel3Bot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/channel3bot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ChatGLM-Spider", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/chatglm-spider", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ChatGPT Agent", "user": true, "operator": "[OpenAI](https://openai.com)", "function": "AI Agents", "description": "ChatGPT Agent is an AI agent created by OpenAI that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf… More info can be found at https://knownagents.com/agents/chatgpt-agent", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "ChatGPT-User", "user": true, "operator": "[OpenAI](https://openai.com)", "function": "AI Assistants", "description": "ChatGPT-User is OpenAI's web crawler that visits websites when ChatGPT users request information. This enables ChatGPT to include links in its responses. More info can be found at https://knownagents.com/agents/chatgpt-user", "respect": "Yes", "frequency": "Only when prompted by a user." }, { "name": "Claude-Code", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "Claude Code is an AI coding agent by Anthropic that can build, debug, and ship code directly from the terminal, handling tasks like codebase onboarding, multi-file edits,… More info can be found at https://knownagents.com/agents/claude-code", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Claude-SearchBot", "user": false, "operator": "[Anthropic](https://www.anthropic.com)", "function": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses.", "description": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses.", "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)", "frequency": "No information provided." }, { "name": "Claude-User", "user": true, "operator": "[Anthropic](https://www.anthropic.com)", "function": "AI Assistants", "description": "Claude-User is dispatched by Anthropic's Claude AI assistant in response to user prompts, when it needs to fetch content to include in its answers. More info can be found at https://knownagents.com/agents/claude-user", "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)", "frequency": "No information provided." }, { "name": "Claude-Web", "user": false, "operator": "Anthropic", "function": "Undocumented AI Agents", "description": "Claude-Web is an AI-related agent operated by Anthropic. It's currently unclear exactly what it's used for, since there's no official documentation. If you can provide more detail, please contact us. More info can be found at https://knownagents.com/agents/claude-web", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ClaudeBot", "user": false, "operator": "[Anthropic](https://www.anthropic.com)", "function": "Scrapes data to train Anthropic's AI products.", "description": "Scrapes data to train LLMs and AI products offered by Anthropic.", "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)", "frequency": "No information provided." }, { "name": "CledaraBot", "user": true, "operator": "Cledara that visits SaaS providers to complete customer-approved administrative tasks, including invoice collection and user managem…", "function": "AI Agents", "description": "CledaraBot is an AI agent operated by Cledara that visits SaaS providers to complete customer-approved administrative tasks, including invoice collection and user managem… More info can be found at https://knownagents.com/agents/cledarabot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Cloudflare-AutoRAG", "user": false, "operator": "[Cloudflare](https://developers.cloudflare.com/autorag)", "function": "Collects data for AI search", "description": "AutoRAG is an all-in-one AI search solution.", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "CloudflareBrowserRenderingCrawler", "user": false, "operator": "Cloudflare that returns rendered website content for research, monitoring, and AI data workflows", "function": "AI Data Providers", "description": "CloudflareBrowserRenderingCrawler is a web crawler operated by Cloudflare that returns rendered website content for research, monitoring, and AI data workflows. More info can be found at https://knownagents.com/agents/cloudflarebrowserrenderingcrawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "CloudVertexBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "CloudVertexBot is a Google-operated crawler available to site owners to request targeted crawls of their own sites for AI training purposes on the Vertex AI platform. More info can be found at https://knownagents.com/agents/cloudvertexbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Code", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "Code (GitHub Copilot) is an AI coding agent that can autonomously plan, build, and execute development tasks, functioning as a collaborative AI pair programmer. More info can be found at https://knownagents.com/agents/code", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Codex", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "Codex is an AI coding agent by OpenAI that helps developers write, review, debug, and ship code. More info can be found at https://knownagents.com/agents/codex", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "cohere-ai", "user": true, "operator": "[Cohere](https://cohere.com)", "function": "Retrieves data to provide responses to user-initiated prompts.", "description": "Retrieves data based on user prompts.", "respect": "Unclear at this time.", "frequency": "Takes action based on user prompts." }, { "name": "cohere-training-data-crawler", "user": false, "operator": "Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products", "function": "AI Data Scrapers", "description": "cohere-training-data-crawler is a web crawler operated by Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products. More info can be found at https://knownagents.com/agents/cohere-training-data-crawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Cotoyogi", "user": false, "operator": "[ROIS](https://ds.rois.ac.jp/en_center8/en_crawler/)", "function": "AI LLM Scraper.", "description": "Scrapes data for AI training in Japanese language.", "respect": "Yes", "frequency": "No information provided." }, { "name": "CragCrawler", "user": false, "operator": "CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services", "function": "AI Data Providers", "description": "CragCrawler is a web scraping bot operated by CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services. The bot is used… More info can be found at https://knownagents.com/agents/cragcrawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Crawl4AI", "user": false, "operator": "Unclear at this time.", "function": "Undocumented AI Agents", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/crawl4ai", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Crawlspace", "user": false, "operator": "[Crawlspace](https://crawlspace.dev)", "function": "AI Data Providers", "description": "Crawlspace is a web crawler platform that fetches and extracts website content for AI agents, RAG applications, and structured data workflows. More info can be found at https://knownagents.com/agents/crawlspace", "respect": "[Yes](https://news.ycombinator.com/item?id=42756654)", "frequency": "Unclear at this time." }, { "name": "Cursor", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "Cursor is an AI coding agent that helps write, edit, and understand code. More info can be found at https://knownagents.com/agents/cursor", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Datenbank Crawler", "user": false, "operator": "Datenbank", "function": "AI Data Scrapers", "description": "Datenbank Crawler is an AI data scraper operated by Datenbank. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/datenbank-crawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "DeepSeekBot", "user": false, "operator": "DeepSeek", "function": "Training language models and improving AI products", "description": "DeepSeekBot is a web crawler used by DeepSeek to train its language models and improve its AI products.", "respect": "No", "frequency": "Unclear at this time." }, { "name": "Devin", "user": true, "operator": "Devin AI", "function": "AI Coding Agents", "description": "Devin is a software engineering AI assistant that can browse websites and perform web-based tasks, functioning as a collaborative AI teammate for engineering teams. More info can be found at https://knownagents.com/agents/devin", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "Diffbot", "user": false, "operator": "[Diffbot](https://www.diffbot.com/)", "function": "AI Data Providers", "description": "Diffbot is a web crawler that extracts and structures website content using AI-powered visual understanding, providing knowledge graph data for applications like market i… More info can be found at https://knownagents.com/agents/diffbot", "respect": "At the discretion of Diffbot users.", "frequency": "Unclear at this time." }, { "name": "Diffbot-User", "user": true, "operator": "[Diffbot](https://www.diffbot.com/)", "function": "AI Assistants", "description": "Diffbot-User is used by requests originating on behalf of a human user browsing a URL using Diffbot software, in response to their input. Documented by Diffbot at https://docs.diffbot.com/docs/does-crawl-respect-robotstxt", "respect": "Yes", "frequency": "Only when prompted by a user." }, { "name": "Dot", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Dot is an OpenAI AI agent that uses a cloud browser to research websites and carry out ongoing tasks for users. More info can be found at https://knownagents.com/agents/dot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "DoubaoBot", "user": false, "operator": "ByteDance", "function": "AI crawler for ByteDance's Doubao AI assistant.", "description": "DoubaoBot is a crawler operated by ByteDance to collect web content for its Doubao AI assistant. Distinct from Bytespider (general/training) and TikTokSpider (video).", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "DuckAssistBot", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "DuckAssistBot is a web crawler that scans websites to collect content for DuckDuckGo's AI-assisted answers feature, which generates brief responses to search queries usin… More info can be found at https://knownagents.com/agents/duckassistbot", "respect": "[Yes](https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/)", "frequency": "Unclear at this time." }, { "name": "Echobot Bot", "user": false, "operator": "Echobox", "function": "AI Data Scrapers", "description": "Echobot Bot is an AI data scraper operated by Echobox. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/echobot-bot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "EchoboxBot", "user": false, "operator": "[Echobox](https://echobox.com)", "function": "Data collection to support AI-powered products.", "description": "Supports company's AI-powered social and email management products.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ElevenlabsBot", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "ElevenlabsBot fetches pages that customers add to or refresh in ElevenLabs agents' knowledge bases, making that content available in conversations. More info can be found at https://knownagents.com/agents/elevenlabsbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ERNIEBot", "user": false, "operator": "Baidu", "function": "Collects public web content for Baidu's ERNIE large language models.", "description": "ERNIEBot is a crawler operated by Baidu to collect public web content used for its ERNIE (Wenxin) large language models.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ExaBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "ExaBot is a web crawler that indexes web content to power Exa's AI search engine and semantic search APIs for AI applications. More info can be found at https://knownagents.com/agents/exabot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ExaSearchBot", "user": false, "operator": "[Exa](https://exa.ai)", "function": "AI Search Crawlers", "description": "ExaSearchBot is a web crawler operated by Exa that discovers and indexes public web pages so their content can be found, retrieved, and cited through Exa. More info can be found at https://knownagents.com/agents/exasearchbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ExaUser", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "ExaUser is Exa's on-demand fetcher that retrieves live website content when a user or an application built on the Exa API requests it. More info can be found at https://knownagents.com/agents/exauser", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "FacebookBot", "user": false, "operator": "Meta/Facebook", "function": "Training language models", "description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically.", "respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)", "frequency": "Up to 1 page per second" }, { "name": "facebookexternalhit", "user": false, "operator": "Meta/Facebook", "function": "Ostensibly only for sharing, but likely used as an AI crawler as well", "description": "Note that excluding FacebookExternalHit will block incorporating OpenGraph data when sharing in social media, including rich links in Apple's Messages app. [According to Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/), its purpose is \"to crawl the content of an app or website that was shared on one of Meta’s family of apps…\". However, see discussions [here](https://github.com/ai-robots-txt/ai.robots.txt/pull/21) and [here](https://github.com/ai-robots-txt/ai.robots.txt/issues/40#issuecomment-2524591313) for evidence to the contrary.", "respect": "[No](https://github.com/ai-robots-txt/ai.robots.txt/issues/40#issuecomment-2524591313)", "frequency": "Unclear at this time." }, { "name": "Factset_spyderbot", "user": false, "operator": "[Factset](https://www.factset.com/ai)", "function": "AI model training.", "description": "Scrapes data for AI training.", "respect": "Unclear at this time.", "frequency": "No information provided." }, { "name": "FirecrawlAgent", "user": false, "operator": "Firecrawl that extracts web content and converts it into structured data for use in LLM and AI applications", "function": "AI Data Providers", "description": "FirecrawlAgent is a web crawler operated by Firecrawl that extracts web content and converts it into structured data for use in LLM and AI applications. More info can be found at https://knownagents.com/agents/firecrawlagent", "respect": "Yes", "frequency": "No information provided." }, { "name": "FriendlyCrawler", "user": false, "operator": "Unknown", "function": "We are using the data from the crawler to build datasets for machine learning experiments.", "description": "Unclear who the operator is; but data is used for training/machine learning.", "respect": "[Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler)", "frequency": "Unclear at this time." }, { "name": "GeistHaus-PageFetcher", "user": true, "operator": "GeistHaus, a company developing AI systems for therapy and psychological assessment", "function": "AI Assistants", "description": "GeistHaus-PageFetcher is a web crawler operated by GeistHaus, a company developing AI systems for therapy and psychological assessment. This bot fetches web pages as part… More info can be found at https://knownagents.com/agents/geisthaus-pagefetcher", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Gemini-Deep-Research", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "Gemini-Deep-Research is the agent responsible for collecting and scanning resources used in Google Gemini's Deep Research feature, which acts as a personal research assis… More info can be found at https://knownagents.com/agents/gemini-deep-research", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Google-Agent", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Google-Agent is used by agents hosted on Google infrastructure to navigate the web and perform actions upon user request. More info can be found at https://knownagents.com/agents/google-agent", "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-agent)", "frequency": "Unclear at this time." }, { "name": "Google-CloudVertexBot", "user": false, "operator": "Google", "function": "Build and manage AI models for businesses employing Vertex AI", "description": "Google-CloudVertexBot crawls sites on the site owners' request when building Vertex AI Agents.", "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "frequency": "No information." }, { "name": "Google-Extended", "user": false, "operator": "Google", "function": "LLM training.", "description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search.", "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "frequency": "No information." }, { "name": "Google-Firebase", "user": false, "operator": "Google", "function": "Used as part of AI apps developed by users of Google's Firebase AI products.", "description": "Supports Google's Firebase AI products.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Google-Gemini-CLI", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "Gemini CLI is an AI coding agent by Google that can query and edit large codebases, generate apps from images or PDFs, and automate complex workflows directly from the te… More info can be found at https://knownagents.com/agents/google-gemini-cli", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Google-NotebookLM", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "Google-NotebookLM is an AI-powered research and note-taking assistant that helps users synthesize information from uploaded sources like documents, transcripts, or web co… More info can be found at https://knownagents.com/agents/google-notebooklm", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "GoogleAgent-Mariner", "user": true, "operator": "Google", "function": "AI Agents", "description": "GoogleAgent-Mariner is an AI agent created by Google that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on … More info can be found at https://knownagents.com/agents/googleagent-mariner", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "GoogleAgent-URLContext", "user": true, "operator": "Google that retrieves web content on behalf of Gemini API users", "function": "AI Assistants", "description": "GoogleAgent-URLContext is a web fetcher operated by Google that retrieves web content on behalf of Gemini API users. When a developer provides a URL as context in a Gemin… More info can be found at https://knownagents.com/agents/googleagent-urlcontext", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "GoogleOther", "user": false, "operator": "Google", "function": "Scrapes data.", "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"", "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "frequency": "No information." }, { "name": "GoogleOther-Image", "user": false, "operator": "Google", "function": "Scrapes data.", "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"", "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "frequency": "No information." }, { "name": "GoogleOther-Video", "user": false, "operator": "Google", "function": "Scrapes data.", "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"", "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "frequency": "No information." }, { "name": "GPTBot", "user": false, "operator": "[OpenAI](https://openai.com)", "function": "Scrapes data to train OpenAI's products.", "description": "Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies.", "respect": "Yes", "frequency": "No information." }, { "name": "Grok Bot", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Grok Bot is a personal AI agent created by xAI that runs on a persistent cloud computer with a browser, filesystem, and terminal. It completes multi-step tasks across app… More info can be found at https://knownagents.com/agents/grok-bot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "GrokAppAndroid", "user": false, "operator": "[xAI](https://x.ai)", "function": "AI Search Crawlers", "description": "Grok AI is an advanced AI assistant built by xAI that leverages real-time web and X data to produce context-aware, multimodal responses.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "GrokBot", "user": false, "operator": "[xAI](https://x.ai)", "function": "AI Search Crawlers", "description": "Used to collect publicly available web content to support the training of Grok AI models.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "HarkBot", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "HarkBot visits web pages on behalf of signed-in Hark users as Hark carries out research, form-filling, and account actions. More info can be found at https://knownagents.com/agents/harkbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "HenkBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "Henkbot crawls the web on behalf of Valyu, an AI search infrastructure provider that indexes content for use in AI-powered retrieval pipelines. More info can be found at https://knownagents.com/agents/henkbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Hunyuan", "user": false, "operator": "[Tencent](https://hunyuan.tencent.com)", "function": "Hunyuan is a training crawler operated by Tencent.", "description": "Hunyuan is a training crawler operated by Tencent. Its job is to read public web pages and feed that content into a machine-learning pipeline that trains future versions of the model.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "iAskBot", "user": false, "operator": "Unclear at this time.", "function": "Undocumented AI Agents", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/iaskbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "iaskspider", "user": false, "operator": "Unclear at this time.", "function": "Undocumented AI Agents", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/iaskspider", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "iaskspider/2.0", "user": false, "operator": "iAsk", "function": "Crawls sites to provide answers to user queries.", "description": "Used to provide answers to user queries.", "respect": "No", "frequency": "Unclear at this time." }, { "name": "ICC-Crawler", "user": false, "operator": "[NICT](https://nict.go.jp)", "function": "Scrapes data to train and support AI technologies.", "description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.", "respect": "Yes", "frequency": "No information." }, { "name": "ImagesiftBot", "user": false, "operator": "[ImageSift](https://imagesift.com)", "function": "ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support their suite of web intelligence products", "description": "Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Their web intelligence products use this index to enable search and retrieval of similar images.", "respect": "[Yes](https://imagesift.com/about)", "frequency": "No information." }, { "name": "imageSpider", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/imagespider", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "img2dataset", "user": false, "operator": "[img2dataset](https://github.com/rom1504/img2dataset)", "function": "Scrapes images for use in LLMs.", "description": "Downloads large sets of images into datasets for LLM training or other purposes.", "respect": "Unclear at this time.", "frequency": "At the discretion of img2dataset users." }, { "name": "Instinct", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Instinct is a personal AI agent that uses a phone and computer to complete everyday tasks such as ordering groceries, booking travel, arranging appointments, and communic… More info can be found at https://knownagents.com/agents/instinct", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ISSCyberRiskCrawler", "user": false, "operator": "[ISS-Corporate](https://iss-cyber.com)", "function": "Scrapes data to train machine learning models.", "description": "Used to train machine learning based models to quantify cyber risk.", "respect": "No", "frequency": "No information." }, { "name": "kagi-fetcher", "user": true, "operator": "Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res…", "function": "AI Assistants", "description": "kagi-fetcher is an AI Assistant operated by Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res… More info can be found at https://knownagents.com/agents/kagi-fetcher", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Kangaroo Bot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://knownagents.com/agents/kangaroo-bot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "KeenableBot", "user": false, "operator": "[Keenable](https://keenable.ai/)", "function": "AI Search Crawlers", "description": "KeenableBot is a web crawler operated by Keenable for indexing web content for AI search.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Kimi-Agent", "user": true, "operator": "Moonshot AI that browses websites while carrying out user-directed research and other tasks", "function": "AI Agents", "description": "Kimi-Agent is a browser-enabled AI agent operated by Moonshot AI that browses websites while carrying out user-directed research and other tasks. More info can be found at https://knownagents.com/agents/kimi-agent", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Kimi-SearchBot", "user": false, "operator": "[Moonshot AI](https://www.moonshot.ai)", "function": "AI Search Crawlers", "description": "Kimi-SearchBot powers Kimi's search features: it analyzes pages for relevance and builds the search index. Documented by Moonshot AI at https://www.kimi.ai/policies/kimi-crawlers", "respect": "[Yes](https://www.kimi.ai/policies/kimi-crawlers)", "frequency": "No information provided." }, { "name": "Kimi-User", "user": true, "operator": "Moonshot AI that fetches web content on behalf of users interacting with Kimi", "function": "AI Assistants", "description": "Kimi-User is a web crawler operated by Moonshot AI that fetches web content on behalf of users interacting with Kimi. When a user asks Kimi to summarize an article or ans… More info can be found at https://knownagents.com/agents/kimi-user", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "KimiBot", "user": false, "operator": "[Moonshot AI](https://www.moonshot.ai)", "function": "AI Data Scrapers", "description": "KimiBot crawls content potentially used to train Kimi's foundation models. Documented by Moonshot AI at https://www.kimi.ai/policies/kimi-crawlers", "respect": "[Yes](https://www.kimi.ai/policies/kimi-crawlers)", "frequency": "No information provided." }, { "name": "KlaviyoAIBot", "user": true, "operator": "[Klaviyo](https://www.klaviyo.com)", "function": "AI Assistants", "description": "KlaviyoAIBot is Klaviyo's web crawler that fetches publicly available pages from domains explicitly connected to user accounts to power the Kai Customer Agent feature. Th… More info can be found at https://knownagents.com/agents/klaviyoaibot", "respect": "[Yes](https://help.klaviyo.com/hc/en-us/articles/40496146232219)", "frequency": "Indexes based on 'change signals' and user configuration." }, { "name": "KunatoCrawler", "user": false, "operator": "Unclear at this time.", "function": "Undocumented AI Agents", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/kunatocrawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "laion-huggingface-processor", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/laion-huggingface-processor", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "LAIONDownloader", "user": false, "operator": "[Large-scale Artificial Intelligence Open Network](https://laion.ai/)", "function": "AI tools and models for machine learning research.", "description": "LAIONDownloader is a bot by LAION, a non-profit organization that provides datasets, tools and models to liberate machine learning research.", "respect": "[No](https://laion.ai/faq/)", "frequency": "Unclear at this time." }, { "name": "LCC", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/lcc", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Lightpanda", "user": false, "operator": "Anyone who downloads the Lightpanda client. Possibly being used by a [Grok-adjacent](https://github.com/lightpanda-io/browser/issues/3156#issuecomment-5217843616) organization's botnet.", "function": "AI Data Scrapers", "description": "Lightpanda is a custom-built headless browser designed for AI and automation.", "respect": "At the [discretion](https://github.com/lightpanda-io/browser/blob/b04c99a9111564ebe06317f644680eda5e3ee83e/src/help.zon#L385) of Lightpanda users.", "frequency": "Defined per-user." }, { "name": "LinerBot", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "LinerBot is the web crawler used by Liner AI assistant to gather information from academic sources and websites to provide accurate answers with line-by-line source citat… More info can be found at https://knownagents.com/agents/linerbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Linguee Bot", "user": false, "operator": "[Linguee](https://www.linguee.com)", "function": "AI powered translation service", "description": "Linguee Bot is a web crawler used by Linguee to gather training data for its AI powered translation service.", "respect": "No", "frequency": "Unclear at this time." }, { "name": "LinkupBot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/linkupbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Manus-User", "user": true, "operator": "Butterfly Effect, a company based in China", "function": "AI Agents", "description": "Manus-User is a browser-enabled AI agent operated by Butterfly Effect, a company based in China. It autonomously navigates websites, interprets content, and carries out m… More info can be found at https://knownagents.com/agents/manus-user", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Meta-ExternalAgent", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly. More info can be found at https://knownagents.com/agents/meta-externalagent", "respect": "[Yes](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)", "frequency": "Unclear at this time." }, { "name": "meta-externalagent", "user": false, "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "function": "Used to train models and improve products.", "description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\"", "respect": "Yes", "frequency": "No information." }, { "name": "Meta-ExternalFetcher", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://knownagents.com/agents/meta-externalfetcher", "respect": "[No](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)", "frequency": "Unclear at this time." }, { "name": "meta-externalfetcher", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "meta-externalfetcher is used by Meta to perform user-initiated fetches of individual links from AI assistant product functions. More info can be found at https://knownagents.com/agents/meta-externalfetcher", "respect": "[No](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)", "frequency": "Unclear at this time." }, { "name": "meta-webindexer", "user": true, "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)", "function": "AI Assistants", "description": "As per their documentation, \"The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\"", "respect": "Unclear at this time.", "frequency": "Unhinged, more than 1 per second." }, { "name": "MistralAI-Index", "user": false, "operator": "Mistral AI", "function": "Indexes web content for Mistral AI's search, used to answer questions in Le Chat. Per Mistral, not used for model training.", "description": "MistralAI-Index is Mistral AI's indexing crawler. It collects web content for Mistral's search index (the source behind Le Chat's answers). Mistral states the content is not used for generative AI training. Documented at https://docs.mistral.ai/robots", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "MistralAI-Training", "user": false, "operator": "[Mistral AI](https://mistral.ai)", "function": "AI Data Scrapers", "description": "MistralAI-Training crawls web content to build training datasets. Documented by Mistral at https://docs.mistral.ai/robots/", "respect": "[Yes](https://docs.mistral.ai/robots/)", "frequency": "No information provided." }, { "name": "MistralAI-User", "user": true, "operator": "Mistral", "function": "AI Assistants", "description": "MistralAI-User is Mistral's AI assistant bot that performs web browsing and data gathering tasks for users in Le Chat, including opening web pages and retrieving informat… More info can be found at https://knownagents.com/agents/mistralai-user", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "MistralAI-User/1.0", "user": true, "operator": "Mistral AI", "function": "Takes action based on user prompts.", "description": "MistralAI-User is for user actions in LeChat. When users ask LeChat a question, it may visit a web page to help answer and include a link to the source in its response.", "respect": "Yes", "frequency": "Only when prompted by a user." }, { "name": "MoonshotBot", "user": false, "operator": "[Kimi](https://www.kimi.ai/)", "function": "MoonshotBot is a training crawler.", "description": "MoonshotBot is a training crawler operated by Kimi. Its job is to read public web pages and feed that content into a machine-learning pipeline that trains future versions of the model.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Mozilla-Tabstack", "user": false, "operator": "Mozilla that performs programmatic, AI-driven interactions with web content through Tabstack", "function": "AI Data Providers", "description": "Mozilla-Tabstack is an AI agent operated by Mozilla that performs programmatic, AI-driven interactions with web content through Tabstack. More info can be found at https://knownagents.com/agents/mozilla-tabstack", "respect": "Yes", "frequency": "On demand via API." }, { "name": "Muse", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Muse is a personal AI agent created by Meta that uses a web browser and connected apps to complete multi-step tasks such as filling out forms, booking travel, sending ema… More info can be found at https://knownagents.com/agents/muse", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "MyCentralAIScraperBot", "user": false, "operator": "Unclear at this time.", "function": "AI data scraper", "description": "Operator and data use is unclear at this time.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "NagetBot", "user": false, "operator": "Naget Inc (founded by Chris Samarinas, headquarter in Amherst, Massachusetts)", "function": "AI data scraper", "description": "'Naget revolutionizes content discovery through an AI-powered ecosystem that transforms how we generate, organize, share, and discover valuable content.' (https://naget.com/) User-agent string links https://naget.ai/bot which yields 404.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "nbot", "user": false, "operator": "[NBot](https://nbot.ai)", "function": "AI Search Crawlers", "description": "NBot uses advanced AI to scan news outlets, industry blogs, newsletters, and social media.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "netEstate Imprint Crawler", "user": false, "operator": "netEstate", "function": "AI Data Scrapers", "description": "netEstate Imprint Crawler is an AI data scraper operated by netEstate. If you think this is incorrect or can provide additional detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/netestate-imprint-crawler", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "newsai", "user": false, "operator": "Unclear at this time.", "function": "AI data scraper", "description": "User-agent string doen't contain an URL and there multiple sites using the newsai brand.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "NotebookLM", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "NotebookLM is an AI-powered research and note-taking assistant that helps users synthesize information from their own uploaded sources, such as documents, transcripts, or web content. It can generate summaries, answer questions, and highlight key themes from the materials you provide, acting like a personalized research companion built on Google's Gemini model. NotebookLM fetches source URLs when users add them to their notebooks, enabling the AI to access and analyze those pages for context and insights. More info can be found at https://knownagents.com/agents/google-notebooklm", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "NovaAct", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Nova Act is an AI agent created by Amazon that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a… More info can be found at https://knownagents.com/agents/novaact", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "OAI-AdsBot", "user": false, "operator": "[OpenAI](https://openai.com)", "function": "Validates and targets ads on ChatGPT.", "description": "OAI-AdsBot visits the landing page of a page submitted as an ad on ChatGPT, to check it against OpenAI's policies and, per OpenAI, to \"determine when it's most relevant to show the ad to users\". OpenAI states it only visits pages submitted as ads and that the data is not used to train foundation models. Documented at https://developers.openai.com/api/docs/bots", "respect": "Unclear at this time.", "frequency": "Only when a page is submitted as an ad." }, { "name": "OAI-SearchBot", "user": false, "operator": "[OpenAI](https://openai.com)", "function": "Search result generation.", "description": "Crawls sites to surface as results in SearchGPT.", "respect": "[Yes](https://platform.openai.com/docs/bots)", "frequency": "No information." }, { "name": "omgili", "user": false, "operator": "[Webz.io](https://webz.io/)", "function": "Data is sold.", "description": "Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training.", "respect": "[Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/)", "frequency": "No information." }, { "name": "omgilibot", "user": false, "operator": "[Webz.io](https://webz.io/)", "function": "Data is sold.", "description": "Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io.", "respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)", "frequency": "No information." }, { "name": "OpenAI", "user": false, "operator": "[OpenAI](https://openai.com)", "function": "Unclear at this time.", "description": "The purpose of this bot is unclear at this time but it is a member of OpenAI's suite of crawlers.", "respect": "Yes", "frequency": "Unclear at this time." }, { "name": "opencode", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "OpenCode is an open-source AI coding agent that helps developers write code from the terminal, IDE, or desktop, supporting multiple LLM providers and local models. More info can be found at https://knownagents.com/agents/opencode", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Operator", "user": true, "operator": "Unclear at this time.", "function": "AI Agents", "description": "Operator is an AI agent created by OpenAI that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://knownagents.com/agents/operator", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "PanguBot", "user": false, "operator": "the Chinese company Huawei", "function": "AI Data Scrapers", "description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://knownagents.com/agents/pangubot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Panscient", "user": false, "operator": "[Panscient](https://panscient.com)", "function": "Data collection and analysis using machine learning and AI.", "description": "Compiles data on businesses and business professionals that is structured using AI and machine learning.", "respect": "[Yes](https://panscient.com/faq.htm)", "frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address." }, { "name": "panscient.com", "user": false, "operator": "[Panscient](https://panscient.com)", "function": "Data collection and analysis using machine learning and AI.", "description": "Compiles data on businesses and business professionals that is structured using AI and machine learning.", "respect": "[Yes](https://panscient.com/faq.htm)", "frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address." }, { "name": "Payhawk Invoice Fetching Agent", "user": true, "operator": "Payhawk that retrieves invoices from supplier websites for users and attaches them to expense records", "function": "AI Agents", "description": "Payhawk Invoice Fetching Agent is a browser agent operated by Payhawk that retrieves invoices from supplier websites for users and attaches them to expense records. More info can be found at https://knownagents.com/agents/payhawk-invoice-fetching-agent", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Perplexity-User", "user": true, "operator": "[Perplexity](https://www.perplexity.ai/)", "function": "AI Assistants", "description": "Perplexity-User supports user actions within Perplexity. When users ask Perplexity a question, it might visit a web page to help provide an accurate answer and include a … More info can be found at https://knownagents.com/agents/perplexity-user", "respect": "[No](https://docs.perplexity.ai/guides/bots)", "frequency": "Only when prompted by a user." }, { "name": "PerplexityBot", "user": false, "operator": "[Perplexity](https://www.perplexity.ai/)", "function": "Search result generation.", "description": "Crawls sites to surface as results in Perplexity.", "respect": "[Yes](https://docs.perplexity.ai/guides/bots)", "frequency": "No information." }, { "name": "PetalBot", "user": false, "operator": "[Huawei](https://huawei.com/)", "function": "Used to provide recommendations in Hauwei assistant and AI search services.", "description": "Operated by Huawei to provide search and AI assistant services.", "respect": "Yes", "frequency": "No explicit frequency provided." }, { "name": "PhindBot", "user": true, "operator": "[phind](https://www.phind.com/)", "function": "AI Assistants", "description": "PhindBot retrieves web pages to support Phind's developer-focused search and answer engine, which uses current web information to answer technical questions. More info can be found at https://knownagents.com/agents/phindbot", "respect": "Unclear at this time.", "frequency": "No explicit frequency provided." }, { "name": "Poggio-Citations", "user": true, "operator": "Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan…", "function": "AI Assistants", "description": "Poggio-Citations is a web crawler operated by Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan… More info can be found at https://knownagents.com/agents/poggio-citations", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Poseidon Research Crawler", "user": false, "operator": "[Poseidon Research](https://www.poseidonresearch.com)", "function": "AI research crawler", "description": "Lab focused on scaling the interpretability research necessary to make better AI systems possible.", "respect": "Unclear at this time.", "frequency": "No explicit frequency provided." }, { "name": "qodercli", "user": true, "operator": "Qoder that helps developers build, debug, and modify software from the terminal", "function": "AI Coding Agents", "description": "qodercli is a command-line AI coding agent operated by Qoder that helps developers build, debug, and modify software from the terminal. More info can be found at https://knownagents.com/agents/qodercli", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "QualifiedBot", "user": true, "operator": "[Qualified](https://www.qualified.com)", "function": "AI Assistants", "description": "QualifiedBot is Qualified's web crawler that analyzes customer websites to provide contextual information for their AI-powered chatbots and conversational marketing platf… More info can be found at https://knownagents.com/agents/qualifiedbot", "respect": "Unclear at this time.", "frequency": "No explicit frequency provided." }, { "name": "Querit-SearchBot", "user": false, "operator": "Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg…", "function": "AI Data Providers", "description": "Querit-SearchBot is a web crawler operated by Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg… More info can be found at https://knownagents.com/agents/querit-searchbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "QueritBot", "user": false, "operator": "Querit, a company providing a search API for large language model integration", "function": "AI Data Providers", "description": "QueritBot is a web crawler operated by Querit, a company providing a search API for large language model integration. This bot indexes web content to power the real-time … More info can be found at https://knownagents.com/agents/queritbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "QuillBot", "user": false, "operator": "[Quillbot](https://quillbot.com)", "function": "Company offers AI detection, writing tools and other services.", "description": "Operated by QuillBot as part of their suite of AI product offerings.", "respect": "Unclear at this time.", "frequency": "No explicit frequency provided." }, { "name": "quillbot.com", "user": false, "operator": "[Quillbot](https://quillbot.com)", "function": "Company offers AI detection, writing tools and other services.", "description": "Operated by QuillBot as part of their suite of AI product offerings.", "respect": "Unclear at this time.", "frequency": "No explicit frequency provided." }, { "name": "QwenBot", "user": false, "operator": "Alibaba", "function": "Collects public web content for Alibaba's Qwen (Tongyi Qianwen) large language models.", "description": "QwenBot is a crawler operated by Alibaba to collect public web content for its Qwen (Tongyi Qianwen) large language models. Alibaba also operates TongyiBot.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Reflectionbot", "user": false, "operator": "[Reflection](https://reflection.ai/)", "function": "Undocumented AI Agents", "description": "An undocumented crawler whose user agent links to Reflection, a company that builds AI models.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "SBIntuitionsBot", "user": false, "operator": "[SB Intuitions](https://www.sbintuitions.co.jp/en/)", "function": "Uses data gathered in AI development and information analysis.", "description": "AI development and information analysis", "respect": "[Yes](https://www.sbintuitions.co.jp/en/bot/)", "frequency": "No information." }, { "name": "Scrapy", "user": false, "operator": "[Zyte](https://www.zyte.com)", "function": "Scrapes data for a variety of uses including training AI.", "description": "\"AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets.\"", "respect": "Unclear at this time.", "frequency": "No information." }, { "name": "SemrushBot-OCOB", "user": false, "operator": "[Semrush](https://www.semrush.com/)", "function": "Crawls your site for ContentShake AI tool.", "description": "Data collected is used for the ContentShake AI tool reports.", "respect": "[Yes](https://www.semrush.com/bot/)", "frequency": "Roughly once every 10 seconds." }, { "name": "SemrushBot-SWA", "user": false, "operator": "[Semrush](https://www.semrush.com/)", "function": "Checks URLs on your site for SEO Writing Assistant.", "description": "Data collected is used for the SEO Writing Assistant tool to check if URL is accessible.", "respect": "[Yes](https://www.semrush.com/bot/)", "frequency": "Roughly once every 10 seconds." }, { "name": "Shap-User", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "Shap-User accesses web content on behalf of users of Parallel Web Systems products. It identifies user-initiated requests rather than automatic web crawling. More info can be found at https://knownagents.com/agents/shap-user", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "ShapBot", "user": false, "operator": "[Parallel](https://parallel.ai)", "function": "AI Data Providers", "description": "ShapBot is a web crawler by Parallel that collects and structures web content to power its search, extraction, and deep research APIs, providing AI agents with high-accur… More info can be found at https://knownagents.com/agents/shapbot", "respect": "[Yes](https://docs.parallel.ai/features/crawler)", "frequency": "Unclear at this time." }, { "name": "Sidetrade indexer bot", "user": false, "operator": "[Sidetrade](https://www.sidetrade.com)", "function": "Extracts data for a variety of uses including training AI.", "description": "AI product training.", "respect": "Unclear at this time.", "frequency": "No information." }, { "name": "Spider", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/spider", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "TavilyBot", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "TavilyBot is a web crawler by Tavily that indexes and extracts content from billions of pages, providing real-time search, extraction, and research data to ground AI agen… More info can be found at https://knownagents.com/agents/tavilybot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Terra Cotta", "user": false, "operator": "Unclear at this time.", "function": "AI Data Providers", "description": "Terra Cotta is Ceramic's web crawler that indexes public content to power their web-scale search API for AI and LLMs. More info can be found at https://knownagents.com/agents/terra-cotta", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "TerraCotta", "user": false, "operator": "[Ceramic AI](https://ceramic.ai/)", "function": "AI Data Providers", "description": "TerraCotta is Ceramic's web crawler that indexes public content to power their web-scale search API for AI and LLMs. More info can be found at https://knownagents.com/agents/terracotta", "respect": "[Yes](https://github.com/CeramicTeam/CeramicTerracotta)", "frequency": "Unclear at this time." }, { "name": "Thinkbot", "user": false, "operator": "[Thinkbot](https://www.thinkbot.agency)", "function": "Insights on AI integration and automation.", "description": "Collects data for analysis on AI usage and automation.", "respect": "No", "frequency": "Unclear at this time." }, { "name": "TikTokSpider", "user": false, "operator": "ByteDance", "function": "LLM training.", "description": "Downloads data to train LLMS, as per Bytespider.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Timpibot", "user": false, "operator": "[Timpi](https://timpi.io)", "function": "Scrapes data for use in training LLMs.", "description": "Makes data available for training AI models.", "respect": "Unclear at this time.", "frequency": "No information." }, { "name": "TongyiBot", "user": true, "operator": "Alibaba that fetches web content for the Tongyi Qianwen assistant and related Qwen-generated answers", "function": "AI Assistants", "description": "TongyiBot is a web crawler operated by Alibaba that fetches web content for the Tongyi Qianwen assistant and related Qwen-generated answers. More info can be found at https://knownagents.com/agents/tongyibot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Trae", "user": true, "operator": "Unclear at this time.", "function": "AI Coding Agents", "description": "Trae is an AI-powered coding agent developed by ByteDance that can understand codebases, fetch web content, and generate code. More info can be found at https://knownagents.com/agents/trae", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "TwinAgent", "user": true, "operator": "Twin, a platform that creates automated workers to perform tasks by integrating with APIs and controlling web applications through browser automa…", "function": "AI Agents", "description": "TwinAgent is operated by Twin, a platform that creates automated workers to perform tasks by integrating with APIs and controlling web applications through browser automa… More info can be found at https://knownagents.com/agents/twinagent", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "UseAI", "user": true, "operator": "Unclear at this time.", "function": "AI Assistants", "description": "UseAI is a web crawler associated with Use AI, a platform that provides an AI workspace where users can chat with AI models, research the web, and perform various tasks. … More info can be found at https://knownagents.com/agents/useai", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "VelenPublicWebCrawler", "user": false, "operator": "[Velen Crawler](https://velen.io)", "function": "Scrapes data for business data sets and machine learning models.", "description": "\"Our goal with this crawler is to build business datasets and machine learning models to better understand the web.\"", "respect": "[Yes](https://velen.io)", "frequency": "No information." }, { "name": "WARDBot", "user": false, "operator": "WEBSPARK", "function": "AI Data Scrapers", "description": "WARDBot is an AI data scraper operated by WEBSPARK. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://knownagents.com/agents/wardbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "Webzio-Extended", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://knownagents.com/agents/webzio-extended", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "webzio-extended", "user": false, "operator": "Unclear at this time.", "function": "AI Data Scrapers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/webzio-extended", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "wpbot", "user": false, "operator": "[QuantumCloud](https://www.quantumcloud.com)", "function": "Live chat support and lead generation.", "description": "wpbot is a used to support the functionality of the AI Chatbot for WordPress plugin. It supports the use of customer models, data collection and customer support.", "respect": "Unclear at this time; opt out provided via [Google Form](https://forms.gle/ajBaxygz9jSR8p8G9)", "frequency": "Unclear at this time." }, { "name": "WRTNBot", "user": false, "operator": "Unclear at this time.", "function": "Undocumented AI Agents", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/wrtnbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "xAI-Grok", "user": false, "operator": "[xAI](https://x.ai/)", "function": "xAI-Grok is an xAI crawler token associated with Grok content collection.", "description": "xAI-Grok is an xAI crawler token associated with Grok content collection. It is classified as training because it performs automated collection rather than a user-triggered fetch.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "YaK", "user": false, "operator": "[Meltwater](https://www.meltwater.com/en/suite/consumer-intelligence)", "function": "According to the [Meltwater Consumer Intelligence page](https://www.meltwater.com/en/suite/consumer-intelligence) 'By applying AI, data science, and market research expertise to a live feed of global ", "description": "Retrieves data used for Meltwater's AI enabled consumer intelligence suite", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "YandexAdditional", "user": false, "operator": "[Yandex](https://yandex.ru)", "function": "Scrapes/analyzes data for the YandexGPT LLM.", "description": "Retrieves data used for YandexGPT quick answers features.", "respect": "[Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en)", "frequency": "No information." }, { "name": "YandexAdditionalBot", "user": false, "operator": "[Yandex](https://yandex.ru)", "function": "Scrapes/analyzes data for the YandexGPT LLM.", "description": "Retrieves data used for YandexGPT quick answers features.", "respect": "[Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en)", "frequency": "No information." }, { "name": "YiBot", "user": false, "operator": "[01.ai](https://01.ai/), an AI company founded by Kai-Fu Lee.", "function": "This bot crawls the web to gather data and content for indexation and AI model training.", "description": "YiBot/1.0 is an advanced artificial intelligence platform designed to enhance interaction and engagement across various applications. Its architecture allows for seamless integration into websites and applications, providing users with a responsive and intuitive interface.", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "YiyanBot", "user": true, "operator": "Baidu that fetches web content for the yiyan", "function": "AI Assistants", "description": "YiyanBot is a web crawler operated by Baidu that fetches web content for the yiyan.baidu.com assistant and related ERNIE-generated answers. More info can be found at https://knownagents.com/agents/yiyanbot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." }, { "name": "YouBot", "user": false, "operator": "[You](https://about.you.com/youchat/)", "function": "Scrapes data for search engine and LLMs.", "description": "Retrieves data used for You.com web search engine and LLMs.", "respect": "[Yes](https://about.you.com/youbot/)", "frequency": "No information." }, { "name": "ZanistaBot", "user": false, "operator": "Unclear at this time.", "function": "AI Search Crawlers", "description": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/zanistabot", "respect": "Unclear at this time.", "frequency": "Unclear at this time." } ] }