Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

21.4 KB
Raw Blame
/**
 * Bot user-agent patterns, vendored from the `isbot` package (Unlicense / public domain)
 * and annotated with categories.
 *
 * Source: https://github.com/omrilotan/isbot/blob/main/src/patterns.json
 *
 * Categories drive policy in trackEvent: some bot categories are dropped silently,
 * others can be counted and tagged. To re-sync against upstream, copy patterns.json
 * verbatim and reapply categories from this file by matching the `pattern` string.
 *
 * IMPORTANT: order matters. classifyUA returns the FIRST matching pattern, so more
 * specific patterns must appear before generic substring matches. The upstream
 * ordering already mostly satisfies this (anchored patterns first, generic
 * substrings last); preserve it on sync.
 */

export type BotCategory =
  | "search" // search engine crawlers (Googlebot, Bingbot, DuckDuckBot, etc.)
  | "ai" // AI training / retrieval / agent crawlers
  | "social" // social link-preview bots (facebookexternalhit, Twitterbot, Slackbot)
  | "monitoring" // uptime / synthetic / performance monitoring
  | "seo" // SEO crawlers (Ahrefs, SEMrush, Moz, Nutch, etc.)
  | "security" // security scanners (Burp, ClamAV, etc.)
  | "framework" // HTTP libraries / scripting clients (curl, python-requests, etc.)
  | "headless" // headless browsers and browser automation
  | "generic"; // matched a bot-ish pattern but uncategorized

/**
 * What the bot is *for*. `category` says which family a pattern belongs to;
 * `purpose` says what the operator does with the fetch, which is the
 * distinction site owners actually care about — a training crawler building a
 * corpus, an answer engine indexing for retrieval, and a human asking an agent
 * to open this page right now are three very different visitors that all share
 * `category: "ai"`.
 *
 * Purpose is additive: `category` is what is persisted historically and what
 * the existing Categories tab reads, so it keeps its meaning unchanged.
 */
export type BotPurpose =
  | "ai_training" // corpus collection for model training
  | "ai_search" // indexing for an AI answer engine
  | "ai_agent" // a human asked an AI to fetch this page, right now
  | "search" // classic search engine indexing
  | "social_preview" // link unfurling
  | "seo" // backlink / rank / site-audit crawlers
  | "monitoring" // uptime, synthetic, performance
  | "security" // scanners
  | "scripted" // HTTP libraries and CLI clients
  | "headless" // browser automation
  | "unknown";

export interface BotPattern {
  /** Regex source string. Compiled with the `i` flag. */
  pattern: string;
  category: BotCategory;
  /**
   * Human-readable bot name, e.g. "GPTBot". Only set on the curated
   * EXTRA_BOT_PATTERNS entries — the vendored upstream list is matched by
   * `pattern` string on re-sync, so it is deliberately left un-annotated.
   */
  name?: string;
  /** Who operates the bot, e.g. "OpenAI". */
  operator?: string;
  purpose?: BotPurpose;
}

export const BOT_PATTERNS: BotPattern[] = [
  { pattern: " daum[ /]", category: "search" },
  { pattern: " deusu/", category: "search" },
  { pattern: "(?:^|[^g])news(?!sapphire)", category: "generic" },
  { pattern: "(?<! (?:channel/|google/))google(?!(app|/google| pixel))", category: "search" },
  { pattern: "(?<! cu)bots?(?:\\b|_)", category: "generic" },
  { pattern: "(?<!(?:lib))http", category: "framework" },
  { pattern: "(?<!cam)scan", category: "security" },
  { pattern: "24x7", category: "monitoring" },
  { pattern: "@[a-z][\\w-]+\\.", category: "generic" },
  { pattern: "\\(\\)", category: "generic" },
  { pattern: "\\.com\\b", category: "generic" },
  { pattern: "\\b\\w+\\.ai", category: "ai" },
  { pattern: "\\bcursor/", category: "ai" },
  { pattern: "\\bmanus-user/", category: "ai" },
  { pattern: "\\bort/", category: "generic" },
  { pattern: "\\bperl\\b", category: "framework" },
  { pattern: "\\bplaywright\\b", category: "headless" },
  { pattern: "\\bsecurityheaders\\b", category: "monitoring" },
  { pattern: "\\bselenium\\b", category: "headless" },
  { pattern: "\\btime/", category: "generic" },
  { pattern: "\\|", category: "generic" },
  { pattern: "^[\\w \\.\\-\\(?:\\):%]+(?:/v?\\d+(?:\\.\\d+)?(?:\\.\\d{1,10})*?)?(?:,|$)", category: "generic" },
  { pattern: "^[\\w\\-]+/[\\w]+$", category: "generic" },
  { pattern: "^[^ ]{50,}$", category: "generic" },
  { pattern: "^\\d+\\b", category: "generic" },
  { pattern: "^\\W", category: "generic" },
  { pattern: "^\\w*search\\b", category: "search" },
  { pattern: "^\\w+/[\\w\\(\\)]*$", category: "generic" },
  { pattern: "^\\w+/\\d\\.\\d\\s\\([\\w@]+\\)$", category: "generic" },
  { pattern: "^active", category: "generic" },
  { pattern: "^ad muncher", category: "generic" },
  { pattern: "^amaya", category: "generic" },
  { pattern: "^apache/", category: "framework" },
  { pattern: "^avsdevicesdk/", category: "generic" },
  { pattern: "^azure", category: "framework" },
  { pattern: "^biglotron", category: "seo" },
  { pattern: "^bot", category: "generic" },
  { pattern: "^bw/", category: "generic" },
  { pattern: "^clamav[ /]", category: "security" },
  { pattern: "^claude-code/", category: "ai" },
  { pattern: "^client/", category: "generic" },
  { pattern: "^cobweb/", category: "seo" },
  { pattern: "^custom", category: "generic" },
  { pattern: "^ddg[_-]android", category: "search" },
  { pattern: "^discourse", category: "generic" },
  { pattern: "^dispatch/\\d", category: "generic" },
  { pattern: "^downcast/", category: "generic" },
  { pattern: "^duckduckgo", category: "search" },
  { pattern: "^email", category: "generic" },
  { pattern: "^facebook", category: "social" },
  { pattern: "^getright/", category: "generic" },
  { pattern: "^gozilla/", category: "generic" },
  { pattern: "^hobbit", category: "generic" },
  { pattern: "^hotzonu", category: "generic" },
  { pattern: "^hwcdn/", category: "generic" },
  { pattern: "^igetter/", category: "generic" },
  { pattern: "^jeode/", category: "framework" },
  { pattern: "^jetty/", category: "framework" },
  { pattern: "^jigsaw", category: "framework" },
  { pattern: "^microsoft bits", category: "framework" },
  { pattern: "^movabletype", category: "generic" },
  { pattern: "^mozilla/\\d\\.\\d\\s[\\w\\.-]+$", category: "generic" },
  { pattern: "^mozilla/\\d\\.\\d\\s\\((?:compatible;)?(?:\\s?[\\w\\d-.]+\\/\\d+\\.\\d+)?\\)$", category: "generic" },
  { pattern: "^navermailapp", category: "generic" },
  { pattern: "^netsurf", category: "generic" },
  { pattern: "^offline", category: "generic" },
  { pattern: "^openai/", category: "ai" },
  { pattern: "^owler", category: "seo" },
  { pattern: "^php", category: "framework" },
  { pattern: "^postman", category: "framework" },
  { pattern: "^python", category: "framework" },
  { pattern: "^rank", category: "seo" },
  { pattern: "^read", category: "generic" },
  { pattern: "^reed", category: "generic" },
  { pattern: "^rest", category: "framework" },
  { pattern: "^rss", category: "generic" },
  { pattern: "^snapchat", category: "social" },
  { pattern: "^space bison", category: "generic" },
  { pattern: "^svn", category: "framework" },
  { pattern: "^swcd ", category: "generic" },
  { pattern: "^taringa", category: "social" },
  { pattern: "^thumbor/", category: "framework" },
  { pattern: "^track", category: "generic" },
  { pattern: "^w3c", category: "generic" },
  { pattern: "^webbandit/", category: "generic" },
  { pattern: "^webcopier", category: "generic" },
  { pattern: "^wget", category: "framework" },
  { pattern: "^whatsapp", category: "social" },
  { pattern: "^wordpress", category: "generic" },
  { pattern: "^xenu link sleuth", category: "seo" },
  { pattern: "^yahoo", category: "search" },
  { pattern: "^yandex", category: "search" },
  { pattern: "^zdm/\\d", category: "generic" },
  { pattern: "^zoom marketplace/", category: "generic" },
  { pattern: "advisor", category: "generic" },
  { pattern: "agent\\b", category: "generic" },
  { pattern: "analyzer", category: "monitoring" },
  { pattern: "archive", category: "generic" },
  { pattern: "ask jeeves/teoma", category: "search" },
  { pattern: "audit", category: "monitoring" },
  { pattern: "bit\\.ly/", category: "generic" },
  { pattern: "bluecoat drtr", category: "generic" },
  { pattern: "browsex", category: "generic" },
  { pattern: "burpcollaborator", category: "security" },
  { pattern: "capture", category: "generic" },
  { pattern: "catch", category: "generic" },
  { pattern: "check\\b", category: "monitoring" },
  { pattern: "checker", category: "monitoring" },
  { pattern: "chrome-lighthouse", category: "monitoring" },
  { pattern: "chromeframe", category: "generic" },
  { pattern: "classifier", category: "generic" },
  { pattern: "cloudflare", category: "generic" },
  { pattern: "convertify", category: "generic" },
  { pattern: "crawl", category: "generic" },
  { pattern: "cypress/", category: "headless" },
  { pattern: "dareboost", category: "monitoring" },
  { pattern: "datanyze", category: "seo" },
  { pattern: "dejaclick", category: "monitoring" },
  { pattern: "detect", category: "monitoring" },
  { pattern: "dmbrowser", category: "generic" },
  { pattern: "download", category: "generic" },
  { pattern: "exaleadcloudview", category: "search" },
  { pattern: "feed", category: "generic" },
  { pattern: "fetcher", category: "generic" },
  { pattern: "firephp", category: "framework" },
  { pattern: "functionize", category: "monitoring" },
  { pattern: "grab", category: "generic" },
  { pattern: "headless", category: "headless" },
  { pattern: "httrack", category: "generic" },
  { pattern: "hubspot marketing grader", category: "monitoring" },
  { pattern: "ibisbrowser", category: "generic" },
  { pattern: "infrawatch", category: "monitoring" },
  { pattern: "insight", category: "monitoring" },
  { pattern: "inspect", category: "monitoring" },
  { pattern: "iplabel", category: "monitoring" },
  { pattern: "java(?!;)", category: "framework" },
  { pattern: "library", category: "generic" },
  { pattern: "linkcheck", category: "seo" },
  { pattern: "mail\\.ru/", category: "search" },
  { pattern: "manager", category: "generic" },
  { pattern: "measure", category: "monitoring" },
  { pattern: "monitor\\b", category: "monitoring" },
  { pattern: "neustar wpm", category: "monitoring" },
  { pattern: "node\\b", category: "framework" },
  { pattern: "nutch", category: "seo" },
  { pattern: "offbyone", category: "generic" },
  { pattern: "onetrust", category: "generic" },
  { pattern: "optimize", category: "monitoring" },
  { pattern: "pageburst", category: "monitoring" },
  { pattern: "pagespeed", category: "monitoring" },
  { pattern: "parser", category: "framework" },
  { pattern: "phantomjs", category: "headless" },
  { pattern: "pingdom", category: "monitoring" },
  { pattern: "powermarks", category: "generic" },
  { pattern: "preview", category: "social" },
  { pattern: "proxy", category: "generic" },
  { pattern: "ptst[ /]\\d", category: "monitoring" },
  { pattern: "retriever", category: "generic" },
  { pattern: "rexx;", category: "framework" },
  { pattern: "rigor", category: "monitoring" },
  { pattern: "rss\\b", category: "generic" },
  { pattern: "scrape", category: "generic" },
  { pattern: "server", category: "generic" },
  { pattern: "sogou", category: "search" },
  { pattern: "sparkler/", category: "generic" },
  { pattern: "speedcurve", category: "monitoring" },
  { pattern: "spider", category: "generic" },
  { pattern: "splash", category: "headless" },
  { pattern: "statuscake", category: "monitoring" },
  { pattern: "supercleaner", category: "generic" },
  { pattern: "synapse", category: "generic" },
  { pattern: "synthetic", category: "monitoring" },
  { pattern: "tools", category: "generic" },
  { pattern: "torrent", category: "generic" },
  { pattern: "transcoder", category: "generic" },
  { pattern: "url", category: "generic" },
  { pattern: "validator", category: "monitoring" },
  { pattern: "virtuoso", category: "framework" },
  { pattern: "wappalyzer", category: "seo" },
  { pattern: "webglance", category: "generic" },
  { pattern: "webkit2png", category: "headless" },
  { pattern: "whatcms/", category: "seo" },
  { pattern: "xtate/", category: "generic" },
];

/**
 * Additional and priority AI / social / monitoring identifiers. Some duplicate
 * upstream patterns intentionally so they win before broad generic rules like
 * "(?<! cu)bots?" and "(?<!(?:lib))http".
 *
 * These are the only entries carrying `name` / `operator` / `purpose`. The
 * upstream BOT_PATTERNS list above is re-synced by matching the `pattern`
 * string against isbot's patterns.json, so annotating it would put that
 * procedure at odds with itself; a named bot that matters belongs here instead,
 * where it also wins the first-match race.
 */
export const EXTRA_BOT_PATTERNS: BotPattern[] = [
  // --- AI: training crawlers -------------------------------------------------
  // Fetch pages to build a corpus. They do not send anyone back to the site.
  { pattern: "\\bgptbot\\b", category: "ai", name: "GPTBot", operator: "OpenAI", purpose: "ai_training" },
  { pattern: "\\bclaudebot\\b", category: "ai", name: "ClaudeBot", operator: "Anthropic", purpose: "ai_training" },
  { pattern: "\\bccbot\\b", category: "ai", name: "CCBot", operator: "Common Crawl", purpose: "ai_training" },
  { pattern: "\\bgoogle-extended\\b", category: "ai", name: "Google-Extended", operator: "Google", purpose: "ai_training" },
  {
    pattern: "\\bapplebot-extended\\b",
    category: "ai",
    name: "Applebot-Extended",
    operator: "Apple",
    purpose: "ai_training",
  },
  { pattern: "\\bbytespider\\b", category: "ai", name: "Bytespider", operator: "ByteDance", purpose: "ai_training" },
  {
    pattern: "\\bmeta-externalagent\\b",
    category: "ai",
    name: "Meta-ExternalAgent",
    operator: "Meta",
    purpose: "ai_training",
  },
  { pattern: "\\bamazonbot\\b", category: "ai", name: "Amazonbot", operator: "Amazon", purpose: "ai_training" },
  { pattern: "\\bcohere-ai\\b", category: "ai", name: "cohere-ai", operator: "Cohere", purpose: "ai_training" },
  { pattern: "\\bdiffbot\\b", category: "ai", name: "Diffbot", operator: "Diffbot", purpose: "ai_training" },
  { pattern: "\\bomgilibot\\b", category: "ai", name: "Omgilibot", operator: "Webz.io", purpose: "ai_training" },

  // --- AI: answer-engine indexers -------------------------------------------
  // Crawl to build a retrieval index that an assistant cites. Unlike training
  // crawlers these can send referral traffic back, which is why they are split
  // out rather than folded into ai_training.
  { pattern: "\\boai-searchbot\\b", category: "ai", name: "OAI-SearchBot", operator: "OpenAI", purpose: "ai_search" },
  {
    pattern: "\\bclaude-searchbot\\b",
    category: "ai",
    name: "Claude-SearchBot",
    operator: "Anthropic",
    purpose: "ai_search",
  },
  {
    pattern: "\\bperplexitybot\\b",
    category: "ai",
    name: "PerplexityBot",
    operator: "Perplexity",
    purpose: "ai_search",
  },
  { pattern: "\\bduckassistbot\\b", category: "ai", name: "DuckAssistBot", operator: "DuckDuckGo", purpose: "ai_search" },
  { pattern: "\\byouchat\\b", category: "ai", name: "YouChat", operator: "You.com", purpose: "ai_search" },
  { pattern: "\\bgrokbot\\b", category: "ai", name: "GrokBot", operator: "xAI", purpose: "ai_search" },

  // --- AI: user-triggered agents --------------------------------------------
  // A person asked an assistant to open this page, so the fetch is one real
  // human's intent rather than background crawling. Highest-value class to
  // separate: these correlate with the AI referral channel, training crawls do
  // not.
  {
    pattern: "\\bchatgpt-user\\b",
    category: "ai",
    name: "ChatGPT-User",
    operator: "OpenAI",
    purpose: "ai_agent",
  },
  { pattern: "\\bclaude-user\\b", category: "ai", name: "Claude-User", operator: "Anthropic", purpose: "ai_agent" },
  {
    pattern: "\\bperplexity-user\\b",
    category: "ai",
    name: "Perplexity-User",
    operator: "Perplexity",
    purpose: "ai_agent",
  },
  {
    pattern: "\\bmistralai-user\\b",
    category: "ai",
    name: "MistralAI-User",
    operator: "Mistral",
    purpose: "ai_agent",
  },
  {
    pattern: "\\bmeta-externalfetcher\\b",
    category: "ai",
    name: "Meta-ExternalFetcher",
    operator: "Meta",
    purpose: "ai_agent",
  },
  { pattern: "\\bgoogle-agent\\b", category: "ai", name: "Google-Agent", operator: "Google", purpose: "ai_agent" },
  { pattern: "^openai/", category: "ai", name: "OpenAI API client", operator: "OpenAI", purpose: "ai_agent" },
  { pattern: "^claude-code/", category: "ai", name: "Claude Code", operator: "Anthropic", purpose: "ai_agent" },
  { pattern: "\\bcursor/", category: "ai", name: "Cursor", operator: "Cursor", purpose: "ai_agent" },
  { pattern: "\\bmanus-user/", category: "ai", name: "Manus", operator: "Manus", purpose: "ai_agent" },
  { pattern: "\\bfirecrawl\\b", category: "ai", name: "Firecrawl", operator: "Firecrawl", purpose: "ai_agent" },

  // Search crawlers whose names can be swallowed by generic "*bot*" rules
  { pattern: "yandexbot", category: "search", name: "YandexBot", operator: "Yandex", purpose: "search" },
  { pattern: "duckduckgo", category: "search", name: "DuckDuckBot", operator: "DuckDuckGo", purpose: "search" },
  { pattern: "slurp", category: "search", name: "Yahoo! Slurp", operator: "Yahoo", purpose: "search" },

  // Social link previewers not caught by upstream
  {
    pattern: "facebookexternalhit",
    category: "social",
    name: "facebookexternalhit",
    operator: "Meta",
    purpose: "social_preview",
  },
  { pattern: "facebot", category: "social", name: "Facebot", operator: "Meta", purpose: "social_preview" },
  { pattern: "twitterbot", category: "social", name: "Twitterbot", operator: "X", purpose: "social_preview" },
  { pattern: "slackbot", category: "social", name: "Slackbot", operator: "Slack", purpose: "social_preview" },
  { pattern: "discordbot", category: "social", name: "Discordbot", operator: "Discord", purpose: "social_preview" },
  { pattern: "linkedinbot", category: "social", name: "LinkedInBot", operator: "LinkedIn", purpose: "social_preview" },
  { pattern: "telegrambot", category: "social", name: "TelegramBot", operator: "Telegram", purpose: "social_preview" },
  {
    pattern: "skypeuripreview",
    category: "social",
    name: "SkypeUriPreview",
    operator: "Microsoft",
    purpose: "social_preview",
  },
  { pattern: "redditbot", category: "social", name: "RedditBot", operator: "Reddit", purpose: "social_preview" },
  { pattern: "pinterestbot", category: "social", name: "Pinterestbot", operator: "Pinterest", purpose: "social_preview" },
  { pattern: "embedly", category: "social", name: "Embedly", operator: "Embedly", purpose: "social_preview" },

  // SEO crawlers commonly seen in the wild
  { pattern: "ahrefsbot", category: "seo", name: "AhrefsBot", operator: "Ahrefs", purpose: "seo" },
  { pattern: "semrushbot", category: "seo", name: "SemrushBot", operator: "Semrush", purpose: "seo" },
  { pattern: "mj12bot", category: "seo", name: "MJ12bot", operator: "Majestic", purpose: "seo" },
  { pattern: "dotbot", category: "seo", name: "DotBot", operator: "Moz", purpose: "seo" },
  { pattern: "rogerbot", category: "seo", name: "rogerbot", operator: "Moz", purpose: "seo" },
  {
    pattern: "screaming frog seo spider",
    category: "seo",
    name: "Screaming Frog",
    operator: "Screaming Frog",
    purpose: "seo",
  },
  { pattern: "serpstatbot", category: "seo", name: "serpstatbot", operator: "Serpstat", purpose: "seo" },

  // Scripted clients and browser automation
  { pattern: "python-requests", category: "framework", name: "python-requests", purpose: "scripted" },
  { pattern: "curl/", category: "framework", name: "curl", purpose: "scripted" },
  { pattern: "^wget", category: "framework", name: "Wget", purpose: "scripted" },
  { pattern: "postmanruntime", category: "framework", name: "Postman", operator: "Postman", purpose: "scripted" },
  { pattern: "apache-httpclient", category: "framework", name: "Apache HttpClient", purpose: "scripted" },
  { pattern: "headlesschrome", category: "headless", name: "HeadlessChrome", purpose: "headless" },
  { pattern: "phantomjs", category: "headless", name: "PhantomJS", purpose: "headless" },
  { pattern: "\\bplaywright\\b", category: "headless", name: "Playwright", purpose: "headless" },
  { pattern: "\\bselenium\\b", category: "headless", name: "Selenium", purpose: "headless" },

  // Monitoring services
  { pattern: "pingdom", category: "monitoring", name: "Pingdom", operator: "Pingdom", purpose: "monitoring" },
  { pattern: "uptimerobot", category: "monitoring", name: "UptimeRobot", operator: "UptimeRobot", purpose: "monitoring" },
  { pattern: "datadog", category: "monitoring", name: "Datadog", operator: "Datadog", purpose: "monitoring" },
  { pattern: "newrelic", category: "monitoring", name: "New Relic", operator: "New Relic", purpose: "monitoring" },
  { pattern: "site24x7", category: "monitoring", name: "Site24x7", operator: "Site24x7", purpose: "monitoring" },
  {
    pattern: "betteruptime",
    category: "monitoring",
    name: "Better Uptime",
    operator: "Better Stack",
    purpose: "monitoring",
  },
  { pattern: "statuscake", category: "monitoring", name: "StatusCake", operator: "StatusCake", purpose: "monitoring" },
  {
    pattern: "chrome-lighthouse",
    category: "monitoring",
    name: "Lighthouse",
    operator: "Google",
    purpose: "monitoring",
  },
];

// EXTRA patterns come first so explicit named bots (ClaudeBot, AhrefsBot, etc.)
// get their specific category before the upstream generic substring matches
// (e.g. the "(?<! cu)bots?" rule) catch them as "generic".
export const ALL_BOT_PATTERNS: BotPattern[] = [...EXTRA_BOT_PATTERNS, ...BOT_PATTERNS];