Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

39.6 KB
Raw Blame
/**
 * Client-side bot detection — mirrors `products.web_analytics.backend.hogql_queries.bot_definitions`
 * and the `getBotName` / `isLikelyBot` / `getTrafficCategory`
 * HogQL functions. Used by real-time dashboards where events arrive via the
 * livestream SSE feed and therefore cannot be classified server-side.
 *
 * Keep this file in sync with `products/web_analytics/backend/hogql_queries/bot_definitions.py`.
 */

export type BotTrafficType = 'AI Agent' | 'Bot' | 'Automation' | 'Regular'

export type BotCategory =
    | 'ai_crawler'
    | 'ai_search'
    | 'ai_assistant'
    | 'search_crawler'
    | 'seo_crawler'
    | 'social_crawler'
    | 'monitoring'
    | 'http_client'
    | 'headless_browser'
    | 'no_user_agent'
    | 'regular'

export interface BotDefinition {
    name: string
    category: BotCategory
    trafficType: BotTrafficType
    operator: string
}

// Ordered by specificity — more specific patterns (e.g. `Applebot-Extended`)
// must precede less specific patterns (`Applebot/`).
export const BOT_DEFINITIONS: { pattern: string; definition: BotDefinition }[] = [
    // AI Crawlers (training data collection)
    {
        pattern: 'GPTBot',
        definition: { name: 'GPTBot', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'OpenAI' },
    },
    {
        pattern: 'Google-CloudVertexBot',
        definition: {
            name: 'Google Cloud Vertex',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'Google',
        },
    },
    {
        pattern: 'Google-Extended',
        definition: { name: 'Google AI', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Google' },
    },
    {
        pattern: 'GoogleOther',
        definition: { name: 'GoogleOther', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Google' },
    },
    {
        pattern: 'Claude-SearchBot',
        definition: { name: 'Claude Search', category: 'ai_search', trafficType: 'AI Agent', operator: 'Anthropic' },
    },
    {
        pattern: 'Claude-User',
        definition: { name: 'Claude User', category: 'ai_assistant', trafficType: 'AI Agent', operator: 'Anthropic' },
    },
    {
        pattern: 'ClaudeBot',
        definition: { name: 'Claude', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Anthropic' },
    },
    {
        pattern: 'Claude-Web',
        definition: { name: 'Claude Web', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Anthropic' },
    },
    {
        pattern: 'anthropic-ai',
        definition: { name: 'Anthropic', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Anthropic' },
    },
    {
        pattern: 'Perplexity-User',
        definition: {
            name: 'Perplexity User',
            category: 'ai_assistant',
            trafficType: 'AI Agent',
            operator: 'Perplexity',
        },
    },
    {
        pattern: 'PerplexityBot',
        definition: { name: 'Perplexity', category: 'ai_search', trafficType: 'AI Agent', operator: 'Perplexity' },
    },
    {
        pattern: 'CCBot',
        definition: { name: 'Common Crawl', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Common Crawl' },
    },
    {
        pattern: 'meta-externalagent',
        definition: { name: 'Meta AI', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Meta' },
    },
    {
        pattern: 'Bytespider',
        definition: { name: 'ByteDance', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'ByteDance' },
    },
    {
        pattern: 'TikTokSpider',
        definition: { name: 'TikTok AI', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'ByteDance' },
    },
    {
        pattern: 'cohere-ai',
        definition: { name: 'Cohere', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Cohere' },
    },
    {
        pattern: 'Diffbot',
        definition: { name: 'Diffbot', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Diffbot' },
    },
    {
        pattern: 'omgili',
        definition: { name: 'Webz.io', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Webz.io' },
    },
    {
        pattern: 'Webzio-Extended',
        definition: { name: 'Webz.io Extended', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Webz.io' },
    },
    {
        pattern: 'Timpibot',
        definition: { name: 'Timpi', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Timpi' },
    },
    {
        pattern: 'Amazonbot',
        definition: { name: 'Amazon', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Amazon' },
    },
    {
        pattern: 'PetalBot',
        definition: { name: 'Petal', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Huawei' },
    },
    {
        pattern: 'Brightbot',
        definition: { name: 'Brightbot', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Bright Data' },
    },
    // AI Search (search result generation)
    {
        pattern: 'OAI-SearchBot',
        definition: { name: 'OpenAI Search', category: 'ai_search', trafficType: 'AI Agent', operator: 'OpenAI' },
    },
    {
        pattern: 'Applebot-Extended',
        definition: { name: 'Apple AI', category: 'ai_search', trafficType: 'AI Agent', operator: 'Apple' },
    },
    // AI Assistants (real-time user-facing fetching)
    {
        pattern: 'ChatGPT-User',
        definition: { name: 'ChatGPT', category: 'ai_assistant', trafficType: 'AI Agent', operator: 'OpenAI' },
    },
    {
        pattern: 'Meta-ExternalFetcher',
        definition: { name: 'Meta Fetcher', category: 'ai_assistant', trafficType: 'AI Agent', operator: 'Meta' },
    },
    {
        pattern: 'DuckAssistBot',
        definition: {
            name: 'DuckDuckGo AI',
            category: 'ai_assistant',
            trafficType: 'AI Agent',
            operator: 'DuckDuckGo',
        },
    },
    {
        pattern: 'MistralAI-User',
        definition: { name: 'Mistral AI', category: 'ai_assistant', trafficType: 'AI Agent', operator: 'Mistral' },
    },
    // Search Crawlers (Applebot/ avoids matching Applebot-Extended)
    {
        pattern: 'Applebot/',
        definition: { name: 'Applebot', category: 'ai_search', trafficType: 'AI Agent', operator: 'Apple' },
    },
    {
        pattern: 'Googlebot',
        definition: { name: 'Googlebot', category: 'search_crawler', trafficType: 'Bot', operator: 'Google' },
    },
    {
        pattern: 'bingbot',
        definition: { name: 'Bingbot', category: 'search_crawler', trafficType: 'Bot', operator: 'Microsoft' },
    },
    {
        pattern: 'Bingbot',
        definition: { name: 'Bingbot', category: 'search_crawler', trafficType: 'Bot', operator: 'Microsoft' },
    },
    {
        pattern: 'YandexBot',
        definition: { name: 'Yandex', category: 'search_crawler', trafficType: 'Bot', operator: 'Yandex' },
    },
    {
        pattern: 'Baiduspider',
        definition: { name: 'Baidu', category: 'search_crawler', trafficType: 'Bot', operator: 'Baidu' },
    },
    {
        pattern: 'DuckDuckBot',
        definition: { name: 'DuckDuckGo', category: 'search_crawler', trafficType: 'Bot', operator: 'DuckDuckGo' },
    },
    {
        pattern: 'Slurp',
        definition: { name: 'Yahoo', category: 'search_crawler', trafficType: 'Bot', operator: 'Yahoo' },
    },
    // SEO Tools
    {
        pattern: 'AhrefsBot',
        definition: { name: 'Ahrefs', category: 'seo_crawler', trafficType: 'Bot', operator: 'Ahrefs' },
    },
    {
        pattern: 'SemrushBot',
        definition: { name: 'Semrush', category: 'seo_crawler', trafficType: 'Bot', operator: 'Semrush' },
    },
    {
        pattern: 'MJ12bot',
        definition: { name: 'Majestic', category: 'seo_crawler', trafficType: 'Bot', operator: 'Majestic' },
    },
    { pattern: 'DotBot', definition: { name: 'Moz', category: 'seo_crawler', trafficType: 'Bot', operator: 'Moz' } },
    // Social Crawlers
    {
        pattern: 'FacebookBot',
        definition: { name: 'Facebook Bot', category: 'social_crawler', trafficType: 'Bot', operator: 'Meta' },
    },
    {
        pattern: 'facebookexternalhit',
        definition: { name: 'Facebook', category: 'social_crawler', trafficType: 'Bot', operator: 'Meta' },
    },
    {
        pattern: 'Twitterbot',
        definition: { name: 'Twitter', category: 'social_crawler', trafficType: 'Bot', operator: 'X' },
    },
    {
        pattern: 'LinkedInBot',
        definition: { name: 'LinkedIn', category: 'social_crawler', trafficType: 'Bot', operator: 'LinkedIn' },
    },
    {
        pattern: 'Pinterest',
        definition: { name: 'Pinterest', category: 'social_crawler', trafficType: 'Bot', operator: 'Pinterest' },
    },
    {
        pattern: 'Slackbot',
        definition: { name: 'Slack', category: 'social_crawler', trafficType: 'Bot', operator: 'Salesforce' },
    },
    {
        pattern: 'TelegramBot',
        definition: { name: 'Telegram', category: 'social_crawler', trafficType: 'Bot', operator: 'Telegram' },
    },
    {
        pattern: 'WhatsApp',
        definition: { name: 'WhatsApp', category: 'social_crawler', trafficType: 'Bot', operator: 'Meta' },
    },
    // Monitoring
    {
        pattern: 'Pingdom',
        definition: { name: 'Pingdom', category: 'monitoring', trafficType: 'Bot', operator: 'SolarWinds' },
    },
    {
        pattern: 'UptimeRobot',
        definition: { name: 'UptimeRobot', category: 'monitoring', trafficType: 'Bot', operator: 'UptimeRobot' },
    },
    {
        pattern: 'Site24x7',
        definition: { name: 'Site24x7', category: 'monitoring', trafficType: 'Bot', operator: 'Zoho' },
    },
    {
        pattern: 'StatusCake',
        definition: { name: 'StatusCake', category: 'monitoring', trafficType: 'Bot', operator: 'StatusCake' },
    },
    {
        pattern: 'Datadog',
        definition: { name: 'Datadog', category: 'monitoring', trafficType: 'Bot', operator: 'Datadog' },
    },
    // HTTP Clients
    {
        pattern: 'curl/',
        definition: { name: 'curl', category: 'http_client', trafficType: 'Automation', operator: 'curl' },
    },
    {
        pattern: 'Wget',
        definition: { name: 'Wget', category: 'http_client', trafficType: 'Automation', operator: 'GNU' },
    },
    {
        pattern: 'python-requests',
        definition: { name: 'Python Requests', category: 'http_client', trafficType: 'Automation', operator: 'Python' },
    },
    {
        pattern: 'axios',
        definition: { name: 'Axios', category: 'http_client', trafficType: 'Automation', operator: 'axios' },
    },
    {
        pattern: 'node-fetch',
        definition: { name: 'Node Fetch', category: 'http_client', trafficType: 'Automation', operator: 'Node.js' },
    },
    {
        pattern: 'Go-http-client',
        definition: { name: 'Go HTTP', category: 'http_client', trafficType: 'Automation', operator: 'Go' },
    },
    {
        pattern: 'okhttp',
        definition: { name: 'OkHttp', category: 'http_client', trafficType: 'Automation', operator: 'Square' },
    },
    {
        pattern: 'Apache-HttpClient',
        definition: { name: 'Apache HTTP', category: 'http_client', trafficType: 'Automation', operator: 'Apache' },
    },
    {
        pattern: 'libwww-perl',
        definition: { name: 'LWP', category: 'http_client', trafficType: 'Automation', operator: 'Perl' },
    },
    {
        pattern: 'Scrapy',
        definition: { name: 'Scrapy', category: 'http_client', trafficType: 'Automation', operator: 'Scrapy' },
    },
    // Headless Browsers
    {
        pattern: 'Lightpanda',
        definition: {
            name: 'Lightpanda',
            category: 'headless_browser',
            trafficType: 'Automation',
            operator: 'Lightpanda',
        },
    },
    {
        pattern: 'HeadlessChrome',
        definition: {
            name: 'Headless Chrome',
            category: 'headless_browser',
            trafficType: 'Automation',
            operator: 'Google',
        },
    },
    {
        pattern: 'PhantomJS',
        definition: {
            name: 'PhantomJS',
            category: 'headless_browser',
            trafficType: 'Automation',
            operator: 'PhantomJS',
        },
    },
    {
        pattern: 'Puppeteer',
        definition: { name: 'Puppeteer', category: 'headless_browser', trafficType: 'Automation', operator: 'Google' },
    },
    {
        pattern: 'Playwright',
        definition: {
            name: 'Playwright',
            category: 'headless_browser',
            trafficType: 'Automation',
            operator: 'Microsoft',
        },
    },
    {
        pattern: 'Selenium',
        definition: { name: 'Selenium', category: 'headless_browser', trafficType: 'Automation', operator: 'Selenium' },
    },
    {
        pattern: 'SlimerJS',
        definition: { name: 'SlimerJS', category: 'headless_browser', trafficType: 'Automation', operator: 'SlimerJS' },
    },
    {
        pattern: 'wkhtmltopdf',
        definition: {
            name: 'wkhtmltopdf',
            category: 'headless_browser',
            trafficType: 'Automation',
            operator: 'wkhtmltopdf',
        },
    },
    // Regex-only impossible-UA rules (Spoofed Edge UA, Truncated WebKit UA, and similar) live
    // only in the server-side `bot_definitions.py`. This livestream mirror matches with
    // `String.includes`, so it can carry plain-token bots but not regex patterns.
    // Self-declared crawlers observed in production `$http_log` traffic
    // AI crawlers
    {
        pattern: 'VioscaleAIBot',
        definition: { name: 'Vioscale AI', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Vioscale' },
    },
    {
        pattern: 'Amzn-SearchBot',
        definition: { name: 'Amazon Search', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Amazon' },
    },
    {
        pattern: 'LeapwaveVIP',
        definition: { name: 'Leapwave', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Leapwave' },
    },
    {
        pattern: 'docs-puller',
        definition: { name: 'docs-puller', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'docs-puller' },
    },
    {
        pattern: 'ToolchestBot',
        definition: { name: 'Toolchest', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Toolchest AI' },
    },
    {
        pattern: 'scopy-docs-crawler',
        definition: { name: 'scopy docs crawler', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'scopy' },
    },
    {
        pattern: 'unstale-collector',
        definition: { name: 'unstale', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'unstale' },
    },
    {
        pattern: 'llms-txt-sync',
        definition: {
            name: 'llms-txt-sync',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'llms-txt-sync',
        },
    },
    {
        pattern: 'PromptingBot',
        definition: { name: 'PromptingBot', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'PromptingBot' },
    },
    {
        pattern: 'llm-code-docs-scraper',
        definition: {
            name: 'LLM code docs scraper',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'llm-code-docs-scraper',
        },
    },
    {
        pattern: 'whichapi-bot',
        definition: { name: 'WhichAPI', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'WhichAPI' },
    },
    {
        pattern: 'Codex-YCB2B',
        definition: {
            name: 'Codex public research',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'OpenAI',
        },
    },
    {
        pattern: 'PostHog-BusinessKnowledge',
        definition: {
            name: 'PostHog BusinessKnowledge',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'PostHog',
        },
    },
    {
        pattern: 'LlmsTxtBot',
        definition: { name: 'LlmsTxtBot', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'llms-txt' },
    },
    {
        pattern: 'every-api/',
        definition: { name: 'Every API', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'every-api' },
    },
    {
        pattern: 'RightAIChoiceBot',
        definition: {
            name: 'Right AI Choice',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'Right AI Choice',
        },
    },
    {
        pattern: 'FlashBrain',
        definition: { name: 'FlashBrain', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Flash Teams' },
    },
    {
        pattern: 'NagetBot',
        definition: { name: 'NagetBot', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Naget' },
    },
    {
        pattern: 'ContextRocketBot',
        definition: {
            name: 'ContextRocket',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'ContextRocket',
        },
    },
    {
        pattern: 'AgentNativeSearchBot',
        definition: {
            name: 'Agent Native Search',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'agent-native-search',
        },
    },
    {
        pattern: 'ElevenlabsCrawler',
        definition: {
            name: 'ElevenLabs Crawler',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'ElevenLabs',
        },
    },
    {
        pattern: 'rag-customer-support-ingest',
        definition: {
            name: 'rag-customer-support-ingest',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'rag-customer-support-ingest',
        },
    },
    {
        pattern: 'OmnichannelWebsiteIngestionBot',
        definition: {
            name: 'Omnichannel Website Ingestion',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'Neurotime',
        },
    },
    {
        pattern: 'llms-txt-check/',
        definition: {
            name: 'llms-txt-check',
            category: 'ai_crawler',
            trafficType: 'AI Agent',
            operator: 'llms-txt-check',
        },
    },
    {
        pattern: 'ref-tools-crawler',
        definition: { name: 'Ref', category: 'ai_crawler', trafficType: 'AI Agent', operator: 'Ref' },
    },
    {
        pattern: 'LinkupBot',
        definition: { name: 'Linkup', category: 'ai_search', trafficType: 'AI Agent', operator: 'Linkup' },
    },
    // Search / index crawlers
    {
        pattern: 'redCactiBot',
        definition: { name: 'redCactiBot', category: 'search_crawler', trafficType: 'Bot', operator: 'redCacti' },
    },
    {
        pattern: 'BelleaireCrawler',
        definition: { name: 'BelleaireCrawler', category: 'search_crawler', trafficType: 'Bot', operator: 'Belleaire' },
    },
    {
        pattern: 'PersimmonDesignResearch',
        definition: {
            name: 'Persimmon Design Research',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Persimmon Software',
        },
    },
    {
        pattern: 'barkReleasesBot',
        definition: { name: 'barkReleasesBot', category: 'search_crawler', trafficType: 'Bot', operator: 'bark' },
    },
    {
        pattern: 'prospector/',
        definition: { name: 'Prospector', category: 'search_crawler', trafficType: 'Bot', operator: 'Rebase' },
    },
    {
        pattern: 'YandexRenderResourcesBot',
        definition: {
            name: 'Yandex Render Resources',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Yandex',
        },
    },
    {
        pattern: 'YandexAccessibilityBot',
        definition: {
            name: 'Yandex Accessibility',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Yandex',
        },
    },
    {
        pattern: 'rewebbot',
        definition: { name: 'rewebbot', category: 'search_crawler', trafficType: 'Bot', operator: 'reweb' },
    },
    {
        pattern: 'Querit-SearchBot',
        definition: { name: 'Querit Search', category: 'search_crawler', trafficType: 'Bot', operator: 'Querit' },
    },
    {
        pattern: 'ImpactVibeDesignResearch',
        definition: {
            name: 'ImpactVibe Design Research',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'ImpactVibe',
        },
    },
    {
        pattern: 'crawl-engine',
        definition: { name: 'crawl-engine', category: 'search_crawler', trafficType: 'Bot', operator: 'Creasource' },
    },
    {
        pattern: 'CompanyFeedServer',
        definition: {
            name: 'CompanyFeedServer',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'company-feed-server',
        },
    },
    {
        pattern: 'BranchenbuchBot',
        definition: {
            name: 'BranchenbuchBot',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Branchenbuch',
        },
    },
    {
        pattern: 'ReachBot',
        definition: { name: 'ReachBot', category: 'search_crawler', trafficType: 'Bot', operator: 'ReachBot' },
    },
    {
        pattern: 'IbouBot',
        definition: { name: 'IbouBot', category: 'search_crawler', trafficType: 'Bot', operator: 'IbouBot' },
    },
    {
        pattern: 'Hypeline',
        definition: { name: 'Hypeline', category: 'search_crawler', trafficType: 'Bot', operator: 'Hypeline' },
    },
    {
        pattern: 'OldSchoolBot',
        definition: { name: 'OldSchoolBot', category: 'search_crawler', trafficType: 'Bot', operator: 'OldSchoolBot' },
    },
    {
        pattern: 'TutusooBot',
        definition: { name: 'TutusooBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Tutusoo' },
    },
    {
        pattern: 'HiringRadarBot',
        definition: { name: 'HiringRadar', category: 'search_crawler', trafficType: 'Bot', operator: 'HiringRadar' },
    },
    {
        pattern: 'slopsearch-linux-crawler',
        definition: { name: 'slopsearch', category: 'search_crawler', trafficType: 'Bot', operator: 'slopsearch' },
    },
    {
        pattern: 'ForgeYourPathAiJobCrawler',
        definition: {
            name: 'ForgeYourPath Job Crawler',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'ForgeYourPath',
        },
    },
    {
        pattern: 'OkaraBot',
        definition: { name: 'OkaraBot', category: 'search_crawler', trafficType: 'Bot', operator: 'OkaraBot' },
    },
    {
        pattern: 'DesignMD-Extract-Bot',
        definition: { name: 'DesignMD Extract', category: 'search_crawler', trafficType: 'Bot', operator: 'DesignMD' },
    },
    {
        pattern: 'LyonlBot',
        definition: { name: 'LyonlBot', category: 'search_crawler', trafficType: 'Bot', operator: 'LyonlBot' },
    },
    {
        pattern: 'Sokjan Crawler',
        definition: { name: 'Sokjan Crawler', category: 'search_crawler', trafficType: 'Bot', operator: 'Sokjan' },
    },
    {
        pattern: 'ScryBot',
        definition: { name: 'ScryBot', category: 'search_crawler', trafficType: 'Bot', operator: 'ScryBot' },
    },
    {
        pattern: 'Heexybot',
        definition: { name: 'Heexybot', category: 'search_crawler', trafficType: 'Bot', operator: 'Heexy' },
    },
    {
        pattern: 'bne.es_bot',
        definition: {
            name: 'Biblioteca Nacional de España',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Biblioteca Nacional de España',
        },
    },
    {
        pattern: 'Zetlyn',
        definition: { name: 'Zetlyn', category: 'search_crawler', trafficType: 'Bot', operator: 'Zetlyn' },
    },
    {
        pattern: 'algorand-platform-newspaper',
        definition: {
            name: 'Algorand platform newspaper',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'algorand.pxke.me',
        },
    },
    {
        pattern: 'VendiditBot',
        definition: { name: 'Vendidit', category: 'search_crawler', trafficType: 'Bot', operator: 'Vendidit' },
    },
    {
        pattern: 'Zee Company Intelligence',
        definition: {
            name: 'Zee Company Intelligence',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Lazyweb',
        },
    },
    {
        pattern: 'NicheStuffBot',
        definition: { name: 'NicheStuff', category: 'search_crawler', trafficType: 'Bot', operator: 'NicheStuff' },
    },
    {
        pattern: 'Axiom-DocBot',
        definition: { name: 'Axiom DocBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Axiom' },
    },
    {
        pattern: 'RealPressBot',
        definition: { name: 'RealPressBot', category: 'search_crawler', trafficType: 'Bot', operator: 'real.press' },
    },
    {
        pattern: 'StraddleLibraryBot',
        definition: { name: 'Straddle Library', category: 'search_crawler', trafficType: 'Bot', operator: 'Straddle' },
    },
    {
        pattern: 'LurantaBot',
        definition: { name: 'Luranta', category: 'search_crawler', trafficType: 'Bot', operator: 'Luranta' },
    },
    {
        pattern: 'UnstenciledBot',
        definition: { name: 'Unstenciled', category: 'search_crawler', trafficType: 'Bot', operator: 'Unstenciled' },
    },
    {
        pattern: 'InloraBot',
        definition: { name: 'Inlora', category: 'search_crawler', trafficType: 'Bot', operator: 'Inlora' },
    },
    {
        pattern: 'EfficientRustCrawler',
        definition: {
            name: 'Efficient Rust Crawler',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Xape',
        },
    },
    {
        pattern: 'DatalenkCreatorResearch',
        definition: {
            name: 'Datalenk Creator Research',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Datalenk',
        },
    },
    {
        pattern: 'Wrzutka-Bot',
        definition: { name: 'Wrzutka', category: 'search_crawler', trafficType: 'Bot', operator: 'Wrzutka' },
    },
    {
        pattern: 'Polon/',
        definition: { name: 'Polon', category: 'search_crawler', trafficType: 'Bot', operator: 'Polon' },
    },
    {
        pattern: 'Babel42Bot',
        definition: { name: 'Babel42', category: 'search_crawler', trafficType: 'Bot', operator: 'Babel42' },
    },
    {
        pattern: 'jscrawler',
        definition: { name: 'jscrawler', category: 'search_crawler', trafficType: 'Bot', operator: 'jscrawler' },
    },
    {
        pattern: 'PolycoreSupabaseDetector',
        definition: {
            name: 'Polycore Supabase Detector',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Polycore',
        },
    },
    {
        pattern: 'UnboundCompute-PublicSnapshot',
        definition: {
            name: 'UnboundCompute Public Snapshot',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'UnboundCompute',
        },
    },
    {
        pattern: 'swissAItalentBot',
        definition: {
            name: 'swissAItalentBot',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'swissaitalent.ch',
        },
    },
    {
        pattern: 'AtlasSearchBot',
        definition: { name: 'AtlasSearch', category: 'search_crawler', trafficType: 'Bot', operator: 'AtlasSearch' },
    },
    {
        pattern: 'KakaoBot',
        definition: { name: 'KakaoBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Kakao' },
    },
    {
        pattern: 'ibou-search',
        definition: { name: 'Ibou Search', category: 'search_crawler', trafficType: 'Bot', operator: 'Ibou' },
    },
    {
        pattern: 'matnx.com/bot',
        definition: { name: 'AtlasBot', category: 'search_crawler', trafficType: 'Bot', operator: 'matnx' },
    },
    {
        pattern: 'DittaBot',
        definition: { name: 'DittaBot', category: 'search_crawler', trafficType: 'Bot', operator: 'DittaBot' },
    },
    {
        pattern: 'T2Bot/',
        definition: { name: 'T2Bot', category: 'search_crawler', trafficType: 'Bot', operator: 'T2' },
    },
    {
        pattern: 'CitePackBot',
        definition: { name: 'CitePack', category: 'search_crawler', trafficType: 'Bot', operator: 'CitePack' },
    },
    {
        pattern: 'ContextFetchBot',
        definition: { name: 'ContextFetch', category: 'search_crawler', trafficType: 'Bot', operator: 'ContextFetch' },
    },
    {
        pattern: 'MachBot/',
        definition: { name: 'MachBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Mach GTM' },
    },
    {
        pattern: 'KERN-OSINT-Crawler',
        definition: { name: 'KERN OSINT Crawler', category: 'search_crawler', trafficType: 'Bot', operator: 'KERN' },
    },
    {
        pattern: 'NCKU-CSIE-WebMining',
        definition: {
            name: 'NCKU CSIE WebMining Crawler',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'NCKU CSIE',
        },
    },
    {
        pattern: 'PoliasBot',
        definition: { name: 'PoliasBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Polias' },
    },
    {
        pattern: 'DevParekhResearch',
        definition: {
            name: 'DevParekhResearch',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'DevParekhResearch',
        },
    },
    {
        pattern: 'TalarionSearchBot',
        definition: { name: 'Talarion Search', category: 'search_crawler', trafficType: 'Bot', operator: 'Talarion' },
    },
    {
        pattern: 'SofyaBot',
        definition: { name: 'SofyaBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Sofya' },
    },
    {
        pattern: 'PkgSeer Documentation Crawler',
        definition: {
            name: 'PkgSeer Documentation Crawler',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'PkgSeer',
        },
    },
    {
        pattern: 'WorklynSearchBot',
        definition: { name: 'Worklyn Search', category: 'search_crawler', trafficType: 'Bot', operator: 'Worklyn' },
    },
    {
        pattern: 'SterkinmatchesLeadgen',
        definition: {
            name: 'Sterkinmatches Leadgen',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Sterkinmatches',
        },
    },
    {
        pattern: 'Brevva/',
        definition: { name: 'Brevva', category: 'search_crawler', trafficType: 'Bot', operator: 'Brevva' },
    },
    {
        pattern: 'FastCrawler/',
        definition: { name: 'FastCrawler', category: 'search_crawler', trafficType: 'Bot', operator: 'FastCrawler' },
    },
    {
        pattern: 'GTMStackResearch',
        definition: { name: 'GTMStack Research', category: 'search_crawler', trafficType: 'Bot', operator: 'GTMStack' },
    },
    {
        pattern: 'JobSearchBot',
        definition: { name: 'JobSearchBot', category: 'search_crawler', trafficType: 'Bot', operator: 'JobSearchBot' },
    },
    {
        pattern: 'Walsingham/',
        definition: { name: 'Walsingham', category: 'search_crawler', trafficType: 'Bot', operator: 'Walsingham' },
    },
    {
        pattern: 'SubBegoneBot',
        definition: { name: 'SubBegoneBot', category: 'search_crawler', trafficType: 'Bot', operator: 'SubBegone' },
    },
    {
        pattern: 'scrape-site/',
        definition: { name: 'scrape-site', category: 'search_crawler', trafficType: 'Bot', operator: 'scrape-site' },
    },
    {
        pattern: 'RevnoxBot',
        definition: { name: 'RevnoxBot', category: 'search_crawler', trafficType: 'Bot', operator: 'Revnox' },
    },
    {
        pattern: 'AlterionLabs/',
        definition: { name: 'AlterionLabs', category: 'search_crawler', trafficType: 'Bot', operator: 'AlterionLabs' },
    },
    {
        pattern: 'usehardal-analytics-crawler',
        definition: {
            name: 'Hardal Analytics Crawler',
            category: 'search_crawler',
            trafficType: 'Bot',
            operator: 'Hardal',
        },
    },
    {
        pattern: 'Yoinka-Events',
        definition: { name: 'Yoinka Events', category: 'search_crawler', trafficType: 'Bot', operator: 'Yoinka' },
    },
    // SEO / marketing crawlers
    {
        pattern: 'LaunchReadyCodeBot',
        definition: {
            name: 'LaunchReadyCodeBot',
            category: 'seo_crawler',
            trafficType: 'Bot',
            operator: 'LaunchReadyCode',
        },
    },
    {
        pattern: 'PricingArchiveBot',
        definition: {
            name: 'PricingArchiveBot',
            category: 'seo_crawler',
            trafficType: 'Bot',
            operator: 'pricing-archive',
        },
    },
    {
        pattern: 'SEOMasterBot',
        definition: { name: 'SEOMasterBot', category: 'seo_crawler', trafficType: 'Bot', operator: 'SEOMaster' },
    },
    {
        pattern: 'CanICancelIt-LinkCheck',
        definition: {
            name: 'CanICancelIt LinkCheck',
            category: 'seo_crawler',
            trafficType: 'Bot',
            operator: 'CanICancelIt',
        },
    },
    {
        pattern: 'serpstatbot',
        definition: { name: 'Serpstat', category: 'seo_crawler', trafficType: 'Bot', operator: 'Serpstat' },
    },
    {
        pattern: 'StackPick',
        definition: { name: 'StackPick', category: 'seo_crawler', trafficType: 'Bot', operator: 'StackPick' },
    },
    {
        pattern: 'UXXRAYBot',
        definition: { name: 'UXXRAY', category: 'seo_crawler', trafficType: 'Bot', operator: 'UXXRAY' },
    },
    {
        pattern: 'NoFluffGraderBot',
        definition: {
            name: 'NoFluff Grader',
            category: 'seo_crawler',
            trafficType: 'Bot',
            operator: 'NoFluff Marketing',
        },
    },
    {
        pattern: 'PagePilot-SiteAudit',
        definition: {
            name: 'PagePilot Site Audit',
            category: 'seo_crawler',
            trafficType: 'Bot',
            operator: 'PagePilot',
        },
    },
    {
        pattern: 'double-ats-customer-discoverer',
        definition: {
            name: 'Double ATS Customer Discoverer',
            category: 'seo_crawler',
            trafficType: 'Bot',
            operator: 'Double',
        },
    },
    {
        pattern: 'BenchRankBot',
        definition: { name: 'BenchRank', category: 'seo_crawler', trafficType: 'Bot', operator: 'BenchRank' },
    },
    {
        pattern: 'QlyzeBot',
        definition: { name: 'Qlyze', category: 'seo_crawler', trafficType: 'Bot', operator: 'Qlyze' },
    },
    {
        pattern: 'AutozellaBot',
        definition: { name: 'Autozella', category: 'seo_crawler', trafficType: 'Bot', operator: 'Autozella' },
    },
    {
        pattern: 'DottpaceAudit',
        definition: { name: 'Dottpace Audit', category: 'seo_crawler', trafficType: 'Bot', operator: 'Dottpace' },
    },
    {
        pattern: 'Clearscopebot',
        definition: { name: 'Clearscope', category: 'seo_crawler', trafficType: 'Bot', operator: 'Clearscope' },
    },
    // Social / link-preview crawlers
    {
        pattern: 'PagePeeker',
        definition: { name: 'PagePeeker', category: 'social_crawler', trafficType: 'Bot', operator: 'PagePeeker' },
    },
    {
        pattern: 'Synapse (bot',
        definition: { name: 'Synapse', category: 'social_crawler', trafficType: 'Bot', operator: 'Matrix.org' },
    },
    {
        pattern: 'Unfurlist',
        definition: { name: 'Unfurlist', category: 'social_crawler', trafficType: 'Bot', operator: 'Doist' },
    },
    // Uptime / monitors / probes
    {
        pattern: 'heap.page source monitor',
        definition: { name: 'heap.page', category: 'monitoring', trafficType: 'Bot', operator: 'heap.page' },
    },
    {
        pattern: 'PulseWatch-Prober',
        definition: { name: 'PulseWatch', category: 'monitoring', trafficType: 'Bot', operator: 'PulseWatch' },
    },
    {
        pattern: 'Scryer/',
        definition: { name: 'Scryer', category: 'monitoring', trafficType: 'Bot', operator: 'Scryer' },
    },
    {
        pattern: 'UnusualAgentAudit',
        definition: { name: 'UnusualAgentAudit', category: 'monitoring', trafficType: 'Bot', operator: 'unusual.ai' },
    },
    {
        pattern: 'VersionWatchBot',
        definition: { name: 'VersionWatch', category: 'monitoring', trafficType: 'Bot', operator: 'VersionWatch' },
    },
    {
        pattern: 'a14y/',
        definition: { name: 'a14y', category: 'monitoring', trafficType: 'Bot', operator: 'a14y' },
    },
    {
        pattern: 'UptimeLensBot',
        definition: { name: 'UptimeLens', category: 'monitoring', trafficType: 'Bot', operator: 'UptimeLens' },
    },
    {
        pattern: 'InterruptIndexBot',
        definition: { name: 'InterruptIndex', category: 'monitoring', trafficType: 'Bot', operator: 'InterruptIndex' },
    },
    {
        pattern: 'CurbCutScanner',
        definition: { name: 'CurbCut Scanner', category: 'monitoring', trafficType: 'Bot', operator: 'CurbCut' },
    },
    {
        pattern: 'VantageBot',
        definition: { name: 'VantageBot', category: 'monitoring', trafficType: 'Bot', operator: 'webapp-monitor' },
    },
    {
        pattern: 'NimbusBlocklistSync',
        definition: { name: 'Nimbus Blocklist Sync', category: 'monitoring', trafficType: 'Bot', operator: 'Nimbus' },
    },
    {
        pattern: 'latency.app/bot',
        definition: { name: 'Latency', category: 'monitoring', trafficType: 'Bot', operator: 'Latency' },
    },
    {
        pattern: 'HonestClaw-Scanner',
        definition: { name: 'HonestClaw Scanner', category: 'monitoring', trafficType: 'Bot', operator: 'HonestClaw' },
    },
    {
        pattern: 'PricingWatchBot',
        definition: {
            name: 'PricingWatchBot',
            category: 'monitoring',
            trafficType: 'Bot',
            operator: 'PricingWatchBot',
        },
    },
    {
        pattern: 'ORBIQ Research Lab',
        definition: { name: 'ORBIQ Research Lab', category: 'monitoring', trafficType: 'Bot', operator: 'ORBIQ' },
    },
    {
        pattern: 'StackClarityWatcher',
        definition: {
            name: 'StackClarity Watcher',
            category: 'monitoring',
            trafficType: 'Bot',
            operator: 'StackClarity',
        },
    },
    {
        pattern: 'RimetryStatusBot',
        definition: { name: 'Rimetry Status', category: 'monitoring', trafficType: 'Bot', operator: 'Rimetry' },
    },
    // HTTP clients
    {
        pattern: 'MrAnandPortfolio',
        definition: { name: 'MrAnandPortfolio', category: 'http_client', trafficType: 'Bot', operator: 'mranand.com' },
    },
    {
        pattern: 'PostHogImageFetcherBot',
        definition: {
            name: 'PostHog Image Fetcher',
            category: 'http_client',
            trafficType: 'Bot',
            operator: 'PostHog',
        },
    },
    {
        pattern: '1nsight-logo-fetcher',
        definition: { name: '1nsight Logo Fetcher', category: 'http_client', trafficType: 'Bot', operator: '1nsight' },
    },
]

export const CATEGORY_LABELS: Record<BotCategory, string> = {
    ai_crawler: 'AI crawler',
    ai_search: 'AI search',
    ai_assistant: 'AI assistant',
    search_crawler: 'Search crawler',
    seo_crawler: 'SEO crawler',
    social_crawler: 'Social crawler',
    monitoring: 'Monitoring',
    http_client: 'HTTP client',
    headless_browser: 'Headless browser',
    no_user_agent: 'No user agent',
    regular: 'Regular',
}

export const detectBot = (userAgent: string | null | undefined): BotDefinition | null => {
    if (userAgent == null || userAgent === '') {
        return null
    }
    for (const { pattern, definition } of BOT_DEFINITIONS) {
        if (userAgent.includes(pattern)) {
            return definition
        }
    }
    return null
}

export const getBotName = (userAgent: string | null | undefined): string => {
    return detectBot(userAgent)?.name ?? ''
}

export const isBot = (userAgent: string | null | undefined): boolean => {
    return detectBot(userAgent) !== null
}