Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.5 KB
Raw Blame
import fs from 'fs'
import path from 'path'

export const KNOWN_BOT_IP_LIST = fs
    .readFileSync(path.join(__dirname, '../../../../', 'assets', 'bot-ips.txt'), 'utf8')
    .split('\n')
    .map((line) => line.trim())

export const KNOWN_BOT_UA_LIST = [
    'bot',
    'crawler',
    'spider',
    'chatgpt-user',
    'claude-user',
    'perplexity-user',
    'mistralai-user',
    'meta-externalfetcher',
    'feedfetcher-google',
    'mediapartners-google',
    'apis-google',
    'slurp',
    'python-urllib',
    'python-requests',
    'aiohttp',
    'httpx',
    'libwww-perl',
    'httpunit',
    'nutch',
    'go-http-client',
    'biglotron',
    'teoma',
    'convera',
    'gigablast',
    'ia_archiver',
    'webmon ',
    'httrack',
    'grub.org',
    'netresearchserver',
    'speedy',
    'fluffy',
    'findlink',
    'panscient',
    'ips-agent',
    'yanga',
    'yandex',
    'yadirectfetcher',
    'cyberpatrol',
    'postrank',
    'page2rss',
    'linkdex',
    'ezooms',
    'heritrix',
    'findthatfile',
    'europarchive.org',
    'mappydata',
    'eright',
    'apercite',
    'aboundex',
    'summify',
    'ec2linkfinder',
    'facebookexternalhit',
    'yeti',
    'retrevopageanalyzer',
    'sogou',
    'wotbox',
    'ichiro',
    'drupact',
    'coccoc',
    'integromedb',
    'siteexplorer.info',
    'proximic',
    'changedetection',
    'cc metadata scaper',
    'g00g1e.net',
    'binlar',
    'a6-indexer',
    'admantx',
    'megaindex',
    'ltx71',
    'bubing',
    'qwantify',
    'lipperhey',
    'addthis',
    'metauri',
    'scrapy',
    'capsulechecker',
    'sonic',
    'sysomos',
    'trove',
    'deadlinkchecker',
    'slack-imgproxy',
    'embedly',
    'iskanie',
    'skypeuripreview',
    'google-adwords-instant',
    'whatsapp',
    'electricmonk',
    'yahoo link preview',
    'xenu link sleuth',
    'pcore-http',
    'appinsights',
    'phantomjs',
    'jetslide',
    'newsharecounts',
    'tineye',
    'linkarchiver',
    'digg deeper',
    'snacktory',
    'okhttp',
    'nuzzel',
    'omgili',
    'pocketparser',
    'um-ln',
    'muckrack',
    'netcraftsurveyagent',
    'appengine-google',
    'jetty',
    'upflow',
    'thinklab',
    'traackr.com',
    'twurly',
    'mastodon',
    'http_get',
    'brandverity',
    'check_http',
    'ezid',
    'genieo',
    'meltwaternews',
    'moreover',
    'scoutjet',
    'seoscanners',
    'hatena',
    'google web preview',
    'adscanner',
    'netvibes',
    'baidu-yunguance',
    'btwebclient',
    'disqus',
    'feedly',
    'fever',
    'flamingo_searchengine',
    'flipboardproxy',
    'g2 web services',
    'vkshare',
    'siteimprove.com',
    'dareboost',
    'feedspot',
    'seokicks',
    'tracemyfile',
    'zgrab',
    'pr-cy.ru',
    'datafeedwatch',
    'zabbix',
    'google-xrawler',
    'axios',
    'amazon cloudfront',
    'pulsepoint',
    'cloudflare-alwaysonline',
    'google-structured-data-testing-tool',
    'wordupinfosearch',
    'webdatastats',
    'httpurlconnection',
    'outbrain',
    'w3c_validator',
    'w3c-checklink',
    'w3c-mobileok',
    'w3c_i18n-checker',
    'feedvalidator',
    'w3c_css_validator',
    'w3c_unicorn',
    'google-physicalweb',
    'blackboard',
    'bazqux',
    'twingly',
    'rivva',
    'dataprovider.com',
    'theoldreader.com',
    'anyevent',
    'nmap scripting engine',
    '2ip.ru',
    'clickagy',
    'google favicon',
    'hubspot',
    'chrome-lighthouse',
    'headlesschrome',
    'simplescraper',
    'fedoraplanet',
    'friendica',
    'nextcloud',
    'tiny tiny rss',
    'datanyze',
    'google-site-verification',
    'trendsmapresolver',
    'tweetedtimes',
    'gwene',
    'simplepie',
    'searchatlas',
    'superfeedr',
    'freewebmonitoring sitechecker',
    'pandalytics',
    'seewithkids',
    'cincraw',
    'freshrss',
    'google-certificates-bridge',
    'viber',
    'evc-batch',
    'virustotal',
    'uptime-kuma',
    'feedbin',
    'snap url preview service',
    'ruxitsynthetic',
    'google-read-aloud',
    'mediapartners',
    'wget',
    'ahrefsgot',
    'ahrefssiteaudit',
    'wesee:search',
    'y!j',
    '[email protected]',
    'deusu',
    'bingpreview',
    'daum',
    'pingdom',
    'barkrowler',
    'yak',
    'ning',
    'ahc',
    'apache-httpclient',
    'buck',
    'newspaper',
    'sentry',
    'fetch',
    'miniflux',
    'validator.nu',
    'grouphigh',
    'checkmarknetwork',
    'www.uptime.com',
    'mixnodecache',
    'domains project',
    'pagepeeker',
    'vigil',
    'php-curl-class',
    'ptst',
    'seostar.co',
]