OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat: bot detection
Carl-Gerhard Lindesvärdf72310c33573
diff --git a/apps/api/src/bots/header-signals.test.ts b/apps/api/src/bots/header-signals.test.tsnew file mode 100644index 000000000..1a90b04c6--- /dev/null+++ b/apps/api/src/bots/header-signals.test.ts@@ -0,0 +1,154 @@+import { describe, expect, it } from 'vitest';+import { detectHeaderAnomalies } from './header-signals';++// Representative UA strings.+const UA = {+ chromeDesktop:+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',+ chromeAndroid:+ 'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',+ edgeDesktop:+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',+ firefox:+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',+ safariMac:+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',+ chromeIOS:+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.0.0 Mobile/15E148 Safari/604.1',+ curl: 'curl/8.4.0',+ goClient: 'Go-http-client/1.1',+};++// Headers a real browser fetch/sendBeacon request carries.+const realBrowserHeaders = {+ 'sec-ch-ua': '"Chromium";v="120"',+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US,en;q=0.9',+};++describe('detectHeaderAnomalies', () => {+ describe('does not flag legitimate browser traffic', () => {+ it('clean desktop Chrome → no reasons', () => {+ expect(+ detectHeaderAnomalies({+ 'user-agent': UA.chromeDesktop,+ ...realBrowserHeaders,+ }),+ ).toEqual([]);+ });++ it('Firefox (no sec-ch-ua, but sends everything else) → no reasons', () => {+ expect(+ detectHeaderAnomalies({+ 'user-agent': UA.firefox,+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US,en;q=0.5',+ }),+ ).toEqual([]);+ });++ it('Safari (no sec-ch-ua, WebKit) → no reasons', () => {+ expect(+ detectHeaderAnomalies({+ 'user-agent': UA.safariMac,+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US,en;q=0.9',+ }),+ ).toEqual([]);+ });++ it('iOS Chrome / CriOS (WebKit, no sec-ch-ua) → not flagged for sec-ch-ua', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.chromeIOS,+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US,en;q=0.9',+ });+ expect(reasons).not.toContain('header:missing_sec_ch_ua');+ expect(reasons).toEqual([]);+ });+ });++ describe('does not apply to non-browser clients', () => {+ for (const [name, ua] of Object.entries({ curl: UA.curl, go: UA.goClient })) {+ it(`${name} → no reasons (handled by isServer/auth, not headers)`, () => {+ expect(detectHeaderAnomalies({ 'user-agent': ua })).toEqual([]);+ });+ }++ it('missing UA entirely → no reasons', () => {+ expect(detectHeaderAnomalies({})).toEqual([]);+ });+ });++ describe('flags spoofed Chromium user agents', () => {+ it('Chrome UA with no sec-ch-ua → missing_sec_ch_ua', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.chromeDesktop,+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US',+ });+ expect(reasons).toContain('header:missing_sec_ch_ua');+ });++ it('Edge UA with no sec-ch-ua → missing_sec_ch_ua', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.edgeDesktop,+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US',+ });+ expect(reasons).toContain('header:missing_sec_ch_ua');+ });++ it('Android Chrome with no sec-ch-ua → missing_sec_ch_ua', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.chromeAndroid,+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US',+ });+ expect(reasons).toContain('header:missing_sec_ch_ua');+ });++ it('bare Chrome UA with no browser headers at all → multiple reasons', () => {+ const reasons = detectHeaderAnomalies({ 'user-agent': UA.chromeDesktop });+ expect(reasons).toContain('header:missing_sec_ch_ua');+ expect(reasons).toContain('header:missing_sec_fetch');+ expect(reasons).toContain('header:missing_accept_language');+ });+ });++ describe('individual header rules', () => {+ it('missing both sec-fetch headers → missing_sec_fetch', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.firefox,+ 'accept-language': 'en-US',+ });+ expect(reasons).toContain('header:missing_sec_fetch');+ });++ it('one sec-fetch header present → no missing_sec_fetch', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.firefox,+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US',+ });+ expect(reasons).not.toContain('header:missing_sec_fetch');+ });++ it('empty-string header counts as absent', () => {+ const reasons = detectHeaderAnomalies({+ 'user-agent': UA.firefox,+ 'sec-fetch-mode': '',+ 'sec-fetch-site': ' ',+ 'accept-language': 'en',+ });+ expect(reasons).toContain('header:missing_sec_fetch');+ });+ });+});diff --git a/apps/api/src/bots/header-signals.ts b/apps/api/src/bots/header-signals.tsnew file mode 100644index 000000000..0db697e7d--- /dev/null+++ b/apps/api/src/bots/header-signals.ts@@ -0,0 +1,83 @@+// Header-consistency bot signals.+//+// Real browsers send a predictable set of request headers. Automated clients+// that spoof a browser User-Agent frequently forget the low-entropy client+// hints (`sec-ch-ua`) and fetch-metadata headers (`sec-fetch-*`) that browsers+// attach automatically. A mismatch between "claims to be Chrome" and "sends+// what Chrome sends" is a strong bot signal.+//+// These checks only MARK traffic (they never block), so the cost of a false+// positive is a mislabeled event. Even so, every rule is deliberately+// UA-conditional to avoid mislabeling legitimate browsers:+// - Firefox and Safari never send `sec-ch-ua` (it is a Blink/Chromium+// feature), so the `sec-ch-ua` rule is gated to desktop/Android Chromium.+// - iOS "Chrome" (CriOS) and other iOS browsers are WebKit under the hood and+// also omit `sec-ch-ua`, so iOS is excluded from that rule too.+//+// Returned reasons are namespaced `header:<detail>` so the caller can count+// distinct signal *categories* when deciding whether to flag an event.++type HeaderValue = string | string[] | undefined;+type Headers = Record<string, HeaderValue>;++function getHeaderString(value: HeaderValue): string {+ if (Array.isArray(value)) {+ return value.join(',');+ }+ return typeof value === 'string' ? value : '';+}++function hasHeader(value: HeaderValue): boolean {+ return getHeaderString(value).trim() !== '';+}++// A browser-shaped UA. Non-browser clients (curl, python-requests, Go-http-+// client) don't send this and are handled elsewhere (server-side auth /+// `isServer`), so header rules simply don't apply to them.+function looksLikeBrowser(ua: string): boolean {+ return ua.includes('Mozilla/');+}++// Desktop/Android Chromium (Blink), which ALWAYS emits `sec-ch-ua` — including+// on `fetch`/`sendBeacon` requests like the OpenPanel web SDK makes. iOS+// browsers (CriOS/EdgiOS/…) are WebKit and excluded, as are Firefox/Safari.+const CHROMIUM_UA_REGEX = /(?:Chrome|Chromium|Edg|OPR)\/\d/;+const APPLE_WEBKIT_UA_REGEX = /(CriOS|EdgiOS|OPiOS|FxiOS|iPhone|iPad|iPod)/;++function isBlinkChromium(ua: string): boolean {+ return CHROMIUM_UA_REGEX.test(ua) && !APPLE_WEBKIT_UA_REGEX.test(ua);+}++export function detectHeaderAnomalies(headers: Headers): string[] {+ const ua = getHeaderString(headers['user-agent']);+ if (!(ua && looksLikeBrowser(ua))) {+ return [];+ }++ const reasons: string[] = [];++ // Blink/Chromium claims to be Chrome but sent no client hints → spoofed UA.+ if (isBlinkChromium(ua) && !hasHeader(headers['sec-ch-ua'])) {+ reasons.push('header:missing_sec_ch_ua');+ }++ // Fetch-metadata headers are sent by all modern browsers on fetch requests.+ // Their total absence points to a raw HTTP client. (Kept general but weak:+ // very old Safari/iOS may omit these, which is why headers only ever count+ // as a single signal category and never flag an event on their own.)+ if (+ !(+ hasHeader(headers['sec-fetch-mode']) ||+ hasHeader(headers['sec-fetch-site'])+ )+ ) {+ reasons.push('header:missing_sec_fetch');+ }++ // Browsers send Accept-Language by default; total absence is mildly suspect.+ if (!hasHeader(headers['accept-language'])) {+ reasons.push('header:missing_accept_language');+ }++ return reasons;+}diff --git a/apps/api/src/bots/suspicion.test.ts b/apps/api/src/bots/suspicion.test.tsnew file mode 100644index 000000000..d6de0e846--- /dev/null+++ b/apps/api/src/bots/suspicion.test.ts@@ -0,0 +1,149 @@+import type { AsnInfo } from '@openpanel/geo';+import { describe, expect, it } from 'vitest';+import {+ applyBotSuspicion,+ stripBotProperties,+ summarizeBotSignals,+} from './suspicion';++describe('stripBotProperties', () => {+ it('removes client-supplied __bot keys in place, keeps the rest', () => {+ const properties: Record<string, unknown> = {+ __bot: '1',+ __bot_reasons: 'forged',+ keep: 'me',+ };+ stripBotProperties(properties);+ expect(properties).toEqual({ keep: 'me' });+ });++ it('tolerates undefined properties', () => {+ expect(() => stripBotProperties(undefined)).not.toThrow();+ });+});++describe('summarizeBotSignals', () => {+ it('no signals → not a bot, empty reasons', () => {+ expect(summarizeBotSignals([])).toEqual({ reasons: '', isBot: false });+ });++ it('single category (datacenter only) → recorded but not flagged', () => {+ const result = summarizeBotSignals(['datacenter_ip:AS15169']);+ expect(result.isBot).toBe(false);+ expect(result.reasons).toBe('datacenter_ip:AS15169');+ });++ it('multiple reasons in the SAME category → still one category, not flagged', () => {+ const result = summarizeBotSignals([+ 'header:missing_sec_ch_ua',+ 'header:missing_sec_fetch',+ 'header:missing_accept_language',+ ]);+ expect(result.isBot).toBe(false);+ });++ it('two distinct categories → flagged', () => {+ const result = summarizeBotSignals([+ 'datacenter_ip:AS16509',+ 'header:missing_sec_ch_ua',+ ]);+ expect(result.isBot).toBe(true);+ expect(result.reasons).toBe(+ 'datacenter_ip:AS16509,header:missing_sec_ch_ua'+ );+ });+});++const HOSTING: AsnInfo = {+ asn: 16_509,+ org: 'Amazon.com, Inc.',+ isHosting: true,+};+const RESIDENTIAL: AsnInfo = { asn: 7922, org: 'Comcast', isHosting: false };++// A spoofed desktop Chrome request with none of the headers Chrome sends.+const spoofedChromeHeaders = {+ 'user-agent':+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',+};+// A clean browser request.+const cleanChromeHeaders = {+ ...spoofedChromeHeaders,+ 'sec-ch-ua': '"Chromium";v="120"',+ 'sec-fetch-mode': 'cors',+ 'sec-fetch-site': 'cross-site',+ 'accept-language': 'en-US,en;q=0.9',+};++describe('applyBotSuspicion', () => {+ it('clean residential browser → no bot props', () => {+ const props = applyBotSuspicion(+ { foo: 'bar' },+ { asnInfo: RESIDENTIAL, headers: cleanChromeHeaders, isServer: false }+ );+ expect(props).toEqual({ foo: 'bar' });+ });++ it('datacenter IP alone → reason recorded, but NOT flagged', () => {+ const props = applyBotSuspicion(+ {},+ { asnInfo: HOSTING, headers: cleanChromeHeaders, isServer: false }+ );+ expect(props?.__bot_reasons).toBe('datacenter_ip:AS16509');+ expect(props?.__bot).toBeUndefined();+ });++ it('datacenter IP + header anomaly → flagged __bot=1', () => {+ const props = applyBotSuspicion(+ {},+ { asnInfo: HOSTING, headers: spoofedChromeHeaders, isServer: false }+ );+ expect(props?.__bot).toBe('1');+ expect(props?.__bot_reasons).toContain('datacenter_ip:AS16509');+ expect(props?.__bot_reasons).toContain('header:missing_sec_ch_ua');+ });++ it('strips client-supplied __bot / __bot_reasons (server-controlled)', () => {+ const props = applyBotSuspicion(+ { __bot: '1', __bot_reasons: 'faked_by_client', keep: 'me' },+ { asnInfo: RESIDENTIAL, headers: cleanChromeHeaders, isServer: false }+ );+ expect(props?.__bot).toBeUndefined();+ expect(props?.__bot_reasons).toBeUndefined();+ expect(props?.keep).toBe('me');+ });++ it('server-side (clientSecretAuth) traffic is never flagged, even from a datacenter', () => {+ const props = applyBotSuspicion(+ {},+ {+ asnInfo: HOSTING,+ headers: spoofedChromeHeaders,+ clientSecretAuth: true,+ isServer: false,+ }+ );+ expect(props).toEqual({});+ });++ it('isServer UA is never flagged, even from a datacenter', () => {+ const props = applyBotSuspicion(+ {},+ {+ asnInfo: HOSTING,+ headers: { 'user-agent': 'Go-http-client/1.1' },+ isServer: true,+ }+ );+ expect(props).toEqual({});+ });++ it('handles undefined properties', () => {+ const props = applyBotSuspicion(undefined, {+ asnInfo: HOSTING,+ headers: spoofedChromeHeaders,+ isServer: false,+ });+ expect(props?.__bot).toBe('1');+ });+});diff --git a/apps/api/src/bots/suspicion.ts b/apps/api/src/bots/suspicion.tsnew file mode 100644index 000000000..cca06fed7--- /dev/null+++ b/apps/api/src/bots/suspicion.ts@@ -0,0 +1,88 @@+// Combines the individual bot signals into a single verdict for an event, and+// applies it to the event's properties.+//+// Each reason is namespaced `<category>:<detail>` (e.g. `datacenter_ip:AS15169`,+// `header:missing_sec_ch_ua`). We flag an event (`__bot=1`) only when at least+// two DISTINCT categories fire — a conservative threshold that avoids+// mislabeling legitimate users who trip a single heuristic (e.g. a corporate+// VPN egressing from a datacenter IP, or an old browser missing a header).+// Multiple reasons within the same category (two header anomalies) count once,+// so a single spoofed request can't inflate itself over the threshold.++import type { AsnInfo } from '@openpanel/geo';+import { detectHeaderAnomalies } from './header-signals';++export const BOT_CATEGORY_THRESHOLD = 2;++export interface BotSuspicion {+ // Comma-joined reasons, safe to store in a ClickHouse Map(String,String)+ // value. Empty string when there are no signals.+ reasons: string;+ // True when >= BOT_CATEGORY_THRESHOLD distinct categories fired.+ isBot: boolean;+}++export function summarizeBotSignals(reasons: string[]): BotSuspicion {+ const categories = new Set(reasons.map((reason) => reason.split(':')[0]));+ return {+ reasons: reasons.join(','),+ isBot: categories.size >= BOT_CATEGORY_THRESHOLD,+ };+}++interface ApplyBotSuspicionOptions {+ asnInfo: AsnInfo;+ headers: Record<string, string | string[] | undefined>;+ // Server-side SDK auth — trusted first-party traffic, never flagged.+ clientSecretAuth?: boolean;+ // Non-browser / server-shaped user agent — trusted, never flagged.+ isServer: boolean;+}++type EventProperties = Record<string, unknown> | undefined;++// `__bot` / `__bot_reasons` are authoritative server-side. Drop any+// client-supplied values wherever properties enter the system (events AND+// profiles), so a client can never smuggle a forged verdict into storage.+export function stripBotProperties(properties: EventProperties): void {+ if (properties) {+ delete properties.__bot;+ delete properties.__bot_reasons;+ }+}++// Annotate an event's properties with bot-suspicion signals WITHOUT blocking it.+// Writes two server-controlled properties: `__bot_reasons` (comma-joined+// granular signals, whenever any fired) and `__bot='1'` (only when the+// conservative multi-category threshold is met). Any client-supplied `__bot*`+// values are always stripped — these keys are authoritative server-side.+// Returns the (possibly newly-created) properties object for the caller to+// assign back.+export function applyBotSuspicion(+ properties: EventProperties,+ { asnInfo, headers, clientSecretAuth, isServer }: ApplyBotSuspicionOptions+): EventProperties {+ stripBotProperties(properties);++ if (clientSecretAuth || isServer) {+ return properties;+ }++ const reasons: string[] = [];+ if (asnInfo.isHosting) {+ reasons.push(`datacenter_ip:AS${asnInfo.asn ?? 'unknown'}`);+ }+ reasons.push(...detectHeaderAnomalies(headers));++ if (reasons.length === 0) {+ return properties;+ }++ const { reasons: reasonsValue, isBot } = summarizeBotSignals(reasons);+ const next = properties ?? {};+ next.__bot_reasons = reasonsValue;+ if (isBot) {+ next.__bot = '1';+ }+ return next;+}diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex cceb4b4e3..a83ed359a 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -1,7 +1,7 @@import { generateId } from '@openpanel/common';import { parseUserAgent } from '@openpanel/common/server';import { getSalts } from '@openpanel/db';-import { getGeoLocation } from '@openpanel/geo';+import { getAsnInfo, getGeoLocation } from '@openpanel/geo';import {type EventsQueuePayloadIncomingEvent,getEventsGroupQueueShard,@@ -11,6 +11,7 @@ import {import type { DeprecatedPostEventPayload } from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';import { getStringHeaders, getTimestamp } from './track.controller';+import { applyBotSuspicion } from '@/bots/suspicion';import { getDeviceId } from '@/utils/ids';export async function postEvent(@@ -33,7 +34,11 @@ export async function postEvent(return;}- const [salts, geo] = await Promise.all([getSalts(), getGeoLocation(ip)]);+ const [salts, geo, asnInfo] = await Promise.all([+ getSalts(),+ getGeoLocation(ip),+ getAsnInfo(ip),+ ]);const { deviceId, sessionId } = await getDeviceId({projectId,ip,@@ -42,6 +47,16 @@ export async function postEvent(});const uaInfo = parseUserAgent(ua, request.body?.properties);+ // Mark (never block) likely bot traffic with __bot / __bot_reasons props.+ // This deprecated route has no body schema, so body can be null/undefined.+ if (request.body) {+ request.body.properties = applyBotSuspicion(request.body.properties, {+ asnInfo,+ headers: request.headers,+ clientSecretAuth: request.clientSecretAuth,+ isServer: uaInfo.isServer,+ });+ }const groupId = uaInfo.isServer? `${projectId}:${request.body?.profileId ?? generateId()}`: deviceId;diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 3b2c14615..3452de4f8 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -10,7 +10,12 @@ import {sessionBuffer,upsertProfile,} from '@openpanel/db';-import { type GeoLocation, getGeoLocation } from '@openpanel/geo';+import {+ type AsnInfo,+ type GeoLocation,+ getAsnInfo,+ getGeoLocation,+} from '@openpanel/geo';import {type EventsQueuePayloadIncomingEvent,getEventsGroupQueueShard,@@ -29,6 +34,7 @@ import type {} from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';import { assocPath, pathOr, pick } from 'ramda';+import { applyBotSuspicion, stripBotProperties } from '@/bots/suspicion';import { HttpError } from '@/utils/errors';import { getDeviceId } from '@/utils/ids';@@ -139,12 +145,18 @@ interface TrackContext {projectId: string;ip: string;ua?: string;+ // Whitelisted subset (getStringHeaders) that ships with the queue payload.headers: Record<string, string | undefined>;+ // Full raw request headers — bot signals need sec-ch-ua / sec-fetch-* /+ // accept-language, which the whitelist above deliberately drops.+ requestHeaders: FastifyRequest['headers'];+ clientSecretAuth: boolean;timestamp: { value: number; isFromPast: boolean };identity?: IIdentifyPayload;deviceId: string;sessionId: string;geo: GeoLocation;+ asnInfo: AsnInfo;}async function buildContext(@@ -175,8 +187,14 @@ async function buildContext(const overrideDeviceId = getOverrideDeviceId(validatedBody);- // Get geo location (needed for track and identify)- const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);+ // Get geo location (needed for track and identify) + ASN (bot detection).+ // Both hit the same MaxMind readers keyed on the same IP and are cached, so+ // resolving them together adds no meaningful latency.+ const [geo, asnInfo, salts] = await Promise.all([+ getGeoLocation(ip),+ getAsnInfo(ip),+ getSalts(),+ ]);const deviceIdResult = await getDeviceId({projectId,@@ -192,6 +210,8 @@ async function buildContext(ip,ua,headers,+ requestHeaders: request.headers,+ clientSecretAuth: request.clientSecretAuth ?? false,timestamp: {value: timestamp.timestamp,isFromPast: timestamp.isTimestampFromThePast,@@ -200,6 +220,7 @@ async function buildContext(deviceId: deviceIdResult.deviceId,sessionId: deviceIdResult.sessionId,geo,+ asnInfo,};}@@ -210,6 +231,13 @@ async function handleTrack(const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);+ // Mark (never block) likely bot traffic with __bot / __bot_reasons props.+ payload.properties = applyBotSuspicion(payload.properties, {+ asnInfo: context.asnInfo,+ headers: context.requestHeaders,+ clientSecretAuth: context.clientSecretAuth,+ isServer: uaInfo.isServer,+ });const groupId = uaInfo.isServer? payload.profileId? `${projectId}:${payload.profileId}`@@ -260,6 +288,8 @@ async function handleIdentify(context: TrackContext): Promise<void> {const { projectId, geo, ua } = context;+ // Profiles must not carry forged bot verdicts either.+ stripBotProperties(payload.properties);const uaInfo = parseUserAgent(ua, payload.properties);await upsertProfile({...payload,diff --git a/packages/geo/package.json b/packages/geo/package.jsonindex 9149e22cd..d2d2e8c45 100644--- a/packages/geo/package.json+++ b/packages/geo/package.json@@ -4,7 +4,8 @@"main": "index.ts","scripts": {"typecheck": "tsc --noEmit",- "codegen": "jiti scripts/download.ts"+ "codegen": "jiti scripts/download.ts && jiti scripts/get-datacenter-asns.ts",+ "gen:asns": "jiti scripts/get-datacenter-asns.ts"},"dependencies": {"@maxmind/geoip2-node": "^6.1.0",diff --git a/packages/geo/scripts/download.ts b/packages/geo/scripts/download.tsindex a7b77912b..379ebb7c2 100644--- a/packages/geo/scripts/download.ts+++ b/packages/geo/scripts/download.ts@@ -8,60 +8,78 @@ const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);import zlib from 'node:zlib';import * as tar from 'tar';-import type { Parser } from 'tar';-const db = 'GeoLite2-City';--const download = async (url: string): Promise<Parser> => {- return new Promise((resolve) => {- https.get(url, (res) => {- const gunzip = zlib.createGunzip();- const parser = tar.t();-- res.pipe(gunzip).pipe(parser as any);- resolve(parser);- });- });-};--async function main(): Promise<void> {- let url = `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${db}.tar.gz`;+// GeoLite2-City → geo lookups; GeoLite2-ASN → datacenter/hosting detection.+const dbs = ['GeoLite2-City', 'GeoLite2-ASN'];+const editionUrl = (db: string): string => {if (process.env.MAXMIND_LICENSE_KEY) {- url = [+ return ['https://download.maxmind.com/app/geoip_download',`?edition_id=${db}&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`,].join('');}+ return `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${db}.tar.gz`;+};- const dest = path.resolve(__dirname, '../');+async function downloadDb(db: string, dest: string): Promise<void> {+ await new Promise<void>((resolve, reject) => {+ const request = https.get(editionUrl(db), (res) => {+ if (res.statusCode !== 200) {+ res.resume();+ reject(new Error(`Download failed for ${db}: HTTP ${res.statusCode}`));+ return;+ }- if (!fs.existsSync(dest)) {- console.log('Geo database not found');- process.exit(1);- }-- try {- const res = await download(url);+ const gunzip = zlib.createGunzip();+ const parser = tar.t();+ // Each extracted .mmdb write must flush before we report success, or an+ // early exit can leave a truncated database on disk.+ const writes: Promise<void>[] = [];- await new Promise<void>((resolve, reject) => {- res.on('entry', (entry) => {+ parser.on('entry', (entry) => {if (entry.path.endsWith('.mmdb')) {const filename = path.join(dest, path.basename(entry.path));- entry.pipe(fs.createWriteStream(filename));+ const file = fs.createWriteStream(filename);+ writes.push(+ new Promise<void>((writeDone, writeFailed) => {+ file.on('finish', writeDone);+ file.on('error', writeFailed);+ })+ );+ entry.pipe(file);console.log('Saved geo database:', filename);+ } else {+ entry.resume();}});- res.on('error', (e) => {- reject(e);+ // pipe() does not forward errors, so each stage needs its own handler.+ gunzip.on('error', reject);+ parser.on('error', reject);+ parser.on('finish', () => {+ Promise.all(writes).then(() => resolve(), reject);});- res.on('finish', () => {- resolve();- });+ res.pipe(gunzip).pipe(parser as any);});++ request.on('error', reject);+ });+}++async function main(): Promise<void> {+ const dest = path.resolve(__dirname, '../');++ if (!fs.existsSync(dest)) {+ console.log('Geo database not found');+ process.exit(1);+ }++ try {+ // The two databases are independent; download them in parallel.+ await Promise.all(dbs.map((db) => downloadDb(db, dest)));} catch (error) {console.error('Error downloading geo database:', error);process.exit(1);diff --git a/packages/geo/scripts/get-datacenter-asns.ts b/packages/geo/scripts/get-datacenter-asns.tsnew file mode 100644index 000000000..e57e26ead--- /dev/null+++ b/packages/geo/scripts/get-datacenter-asns.ts@@ -0,0 +1,68 @@+import fs from 'node:fs';+import { dirname } from 'node:path';+import path from 'node:path';+import { fileURLToPath } from 'node:url';++const __filename = fileURLToPath(import.meta.url);+const __dirname = dirname(__filename);++// Upstream list of Autonomous System Numbers that belong to datacenters /+// hosting providers (AWS, GCP, Azure, OVH, Hetzner, DigitalOcean, …). Traffic+// originating from these ASNs is almost never a real end user browsing a site;+// it is scrapers, monitors and automated clients running on rented compute.+// We resolve an IP's ASN with MaxMind's GeoLite2-ASN db and check membership in+// this set. The list is regenerated (never hand-edited) so upstream additions+// flow in on every `pnpm codegen`.+const SOURCE_URL =+ 'https://raw.githubusercontent.com/X4BNet/lists_vpn/main/input/datacenter/ASN.txt';++// Each line looks like: `AS14061 # DIGITALOCEAN-ASN - DigitalOcean, LLC, US`.+// Blank lines and anything that is not an `AS<number>` entry are ignored.+const AS_LINE_REGEX = /^AS(\d+)\b/;++export function parseAsnList(raw: string): number[] {+ const asns = new Set<number>();+ for (const line of raw.split('\n')) {+ const match = line.trim().match(AS_LINE_REGEX);+ if (match) {+ asns.add(Number(match[1]));+ }+ }+ return [...asns].sort((a, b) => a - b);+}++async function main(): Promise<void> {+ const raw = await fetch(SOURCE_URL).then((res) => res.text());+ const asns = parseAsnList(raw);++ if (asns.length === 0) {+ throw new Error('Parsed 0 datacenter ASNs — refusing to write empty list');+ }++ fs.writeFileSync(+ path.resolve(__dirname, '../src/datacenter-asns.ts'),+ [+ '// This file is generated by the script get-datacenter-asns.ts',+ '',+ `// Source: ${SOURCE_URL}`,+ '',+ '// Autonomous System Numbers belonging to datacenters / hosting providers.',+ '// Used to flag (not block) traffic that is unlikely to be a real end user.',+ `const datacenterAsns: readonly number[] = ${JSON.stringify(asns)};`,+ 'export default datacenterAsns;',+ '',+ ].join('\n'),+ 'utf-8',+ );++ console.log(`✅ Generated datacenter-asns.ts with ${asns.length} ASNs`);+}++// Only fetch + regenerate when run directly, so `parseAsnList` can be imported+// and unit-tested without a network call.+if (process.argv[1] === __filename) {+ main().catch((error) => {+ console.error('Error generating datacenter ASN list:', error);+ process.exit(1);+ });+}diff --git a/packages/geo/src/datacenter-asns.test.ts b/packages/geo/src/datacenter-asns.test.tsnew file mode 100644index 000000000..a3fcf7da6--- /dev/null+++ b/packages/geo/src/datacenter-asns.test.ts@@ -0,0 +1,80 @@+/**+ * Tests for the generated datacenter/hosting ASN list used to flag likely-bot+ * traffic. These exercise the pure list membership (no MaxMind db needed, which+ * is git-ignored and only present after `pnpm codegen`), plus the parser that+ * generates the list. They guard two things:+ * 1. Known hosting providers (AWS, GCP, Azure, OVH, …) stay classified as+ * datacenters, so a regen from upstream can never silently drop them.+ * 2. Residential ISPs (Comcast, AT&T, …) are NOT in the list, so we never+ * mark real end users as bots.+ */++import { describe, expect, it } from 'vitest';+import { parseAsnList } from '../scripts/get-datacenter-asns';+import datacenterAsns from './datacenter-asns';++describe('datacenter ASN list', () => {+ const asnSet = new Set(datacenterAsns);++ describe('classifies known hosting providers as datacenters', () => {+ const hosting: Record<string, number> = {+ Google: 15169,+ Amazon: 16509,+ Azure: 8075,+ OVH: 16276,+ Hetzner: 24940,+ DigitalOcean: 14061,+ Linode: 63949,+ Vultr: 20473,+ };++ for (const [name, asn] of Object.entries(hosting)) {+ it(`flags ${name} (AS${asn})`, () => {+ expect(asnSet.has(asn)).toBe(true);+ });+ }+ });++ describe('does not flag residential ISPs', () => {+ const residential: Record<string, number> = {+ Comcast: 7922,+ 'AT&T': 7018,+ Verizon: 701,+ 'Deutsche Telekom': 3320,+ Telia: 3301,+ };++ for (const [name, asn] of Object.entries(residential)) {+ it(`treats ${name} (AS${asn}) as a real user`, () => {+ expect(asnSet.has(asn)).toBe(false);+ });+ }+ });++ it('is non-empty and contains only positive integers', () => {+ expect(datacenterAsns.length).toBeGreaterThan(100);+ for (const asn of datacenterAsns) {+ expect(Number.isInteger(asn)).toBe(true);+ expect(asn).toBeGreaterThan(0);+ }+ });+});++describe('parseAsnList', () => {+ it('parses `AS<number>` lines and ignores comments/blanks', () => {+ const raw = [+ 'AS14061 # DIGITALOCEAN-ASN - DigitalOcean, LLC, US',+ 'AS16509 # AMAZON-02, US',+ '',+ '# a stray comment line',+ 'not-an-asn-line',+ 'AS15169',+ ].join('\n');++ expect(parseAsnList(raw)).toEqual([14061, 15169, 16509]);+ });++ it('de-duplicates and sorts ascending', () => {+ expect(parseAsnList('AS20\nAS10\nAS20\nAS10')).toEqual([10, 20]);+ });+});diff --git a/packages/geo/src/datacenter-asns.ts b/packages/geo/src/datacenter-asns.tsnew file mode 100644index 000000000..ffc0168ae--- /dev/null+++ b/packages/geo/src/datacenter-asns.ts@@ -0,0 +1,8 @@+// This file is generated by the script get-datacenter-asns.ts++// Source: https://raw.githubusercontent.com/X4BNet/lists_vpn/main/input/datacenter/ASN.txt++// Autonomous System Numbers belonging to datacenters / hosting providers.+// Used to flag (not block) traffic that is unlikely to be a real end user.+const datacenterAsns: readonly number[] = [1442,3223,3561,3722,3842,4250,4323,4694,4851,5435,5577,6188,6698,6718,6724,6764,6870,6939,7203,7349,7489,7506,7595,7598,7850,7979,8075,8100,8455,8556,8560,8849,8972,9009,9166,9290,9370,9412,9667,9823,9925,10200,10207,10297,10439,10532,10929,11230,11235,11274,11588,11831,11878,12200,12222,12586,12617,12824,12876,12989,13209,13213,13378,13647,13739,13909,13926,13955,14061,14120,14127,14160,14244,14384,14415,14442,14567,14576,14618,14708,14956,14986,14987,14992,15003,15083,15169,15189,15391,15395,15497,15510,15626,15694,15734,15919,15967,16125,16262,16276,16284,16397,16509,16535,16625,16628,16652,16862,16973,17019,17216,17439,17669,17881,17918,17920,17971,18120,18450,18570,18779,18978,19084,19133,19148,19234,19318,19437,19531,19624,19844,19871,19969,19994,20021,20068,20248,20264,20401,20450,20454,20473,20552,20598,20692,20738,20773,20836,20860,20940,21100,21159,21217,21321,21342,21499,21501,21859,22152,22363,22552,22611,22720,22781,22903,23033,23052,23108,23273,23342,23352,23470,23535,23881,24220,24319,24381,24482,24549,24558,24611,24679,24725,24768,24806,24875,24931,24940,24958,24961,24971,24997,25048,25128,25163,25260,25291,25369,25379,25532,25642,25780,25820,25926,26277,26347,26481,26484,26496,26548,26978,27175,27223,27229,27257,27357,27411,27589,27597,27640,28099,28216,28333,28682,28747,28753,28855,28997,29066,29067,29073,29097,29119,29140,29182,29302,29311,29331,29354,29452,29522,29550,29593,29691,29713,29748,29802,29838,29854,29869,29883,30083,30152,30176,30235,30475,30633,30693,30836,30849,30900,30998,31034,31103,31240,31242,31472,31590,31659,31698,31898,31981,32097,32181,32244,32275,32306,32338,32400,32475,32489,32613,32647,32740,32780,32911,33070,33083,33182,33251,33260,33302,33322,33330,33387,33438,33480,33552,33569,33696,33724,33785,33891,33905,34011,34164,34305,34432,34541,34649,34745,34927,34971,34989,35017,35048,35278,35295,35366,35415,35470,35540,35662,35908,35913,35914,35916,35974,35994,36007,36024,36114,36183,36236,36290,36351,36352,36408,36536,36666,36791,36887,36970,37018,37088,37153,37170,37209,37230,37248,37269,37280,37308,37347,37377,37472,37506,37518,37521,37540,37643,37661,37692,37714,38001,38107,38279,38894,39020,39326,39351,39392,39451,39458,39486,39566,39572,39704,39756,39832,39839,40156,40244,40281,40374,40438,40476,40539,40676,40715,40728,40819,40824,40861,41062,41079,41369,41406,41427,41562,41653,41665,41828,42120,42160,42210,42244,42311,42331,42399,42400,42418,42442,42465,42473,42532,42612,42622,42675,42695,42699,42705,42708,42730,42776,42831,42927,43021,43128,43146,43198,43289,43317,43350,43472,43541,43620,43962,44050,44066,44144,44398,44477,44803,44901,45090,45102,45152,45179,45187,45201,45470,45481,45486,45577,45671,45693,45815,45887,46177,46260,46261,46430,46433,46475,46516,46562,46606,46664,46805,46816,46844,46873,46945,47143,47161,47172,47205,47223,47328,47385,47447,47544,47549,47577,47583,47588,47625,48031,48093,48282,48446,48505,48812,48825,48894,48896,49313,49349,49367,49453,49485,49505,49532,49544,49693,49815,49834,49949,49981,50297,50465,50608,50613,50655,50673,50835,50872,50915,50926,50968,50986,51050,51109,51159,51167,51191,51241,51248,51290,51294,51395,51430,51698,51731,51765,51790,51852,51862,52000,52048,52173,52219,52236,52270,52321,52335,52347,52465,52468,52674,52925,53013,53055,53057,53101,53221,53225,53281,53332,53340,53342,53370,53559,53589,53597,53667,53755,53850,53889,53914,53918,54104,54203,54290,54334,54455,54489,54500,54527,54540,54555,54641,54817,54825,54839,55051,55225,55229,55286,55293,55536,55720,55761,55799,55933,55967,55990,56106,56110,56322,56617,56630,56635,56732,56784,56799,56934,57043,57169,57230,57286,57345,57363,57367,57433,57669,57682,57752,57773,57858,57879,58065,58073,58113,58305,58667,58683,58797,58922,58936,59135,59253,59349,59432,59472,59504,59554,59615,59632,59677,59705,59711,59729,59764,59787,59791,59795,59816,59854,60011,60068,60117,60118,60253,60362,60404,60435,60476,60485,60505,60558,60567,60713,60729,60739,60781,60800,60973,61046,61102,61107,61138,61147,61157,61280,61317,61412,61440,62005,62026,62049,62071,62082,62088,62217,62240,62282,62310,62370,62471,62540,62563,62567,62605,62651,62756,62838,62899,62904,63008,63018,63119,63128,63129,63199,63213,63473,63567,63916,63949,64022,64245,64249,64286,64484,132070,132071,132203,132225,132425,132509,132717,132779,132816,132869,133120,133143,133229,133296,133393,133480,133499,133752,134351,134450,134451,134963,135377,135822,136052,136258,136557,136907,136988,137409,138608,141039,141995,142036,147049,149020,152390,196645,196678,196745,196827,197155,197226,197328,197372,197395,197439,197540,197648,197706,197902,197914,198047,198153,198171,198310,198313,198347,198414,198432,198524,198605,198651,198968,199129,199213,199481,199653,199733,199785,199847,199883,199990,199997,200000,200019,200039,200088,200147,200350,200532,200904,201011,201200,201449,201525,201553,201597,201630,201634,201670,201702,201709,201814,201862,201983,202015,202023,202053,202118,202276,202448,202836,203020,203417,203523,203629,203758,204196,204287,204601,204957,205016,205053,205544,206092,206216,206246,206589,206898,207990,208323,208566,208795,208969,210329,210558,210731,210779,211619,212238,212317,212861,213010,213230,213887,214036,214574,215859,216071,262170,262287,262603,262978,262990,263032,263093,263237,265620,327705,327784,327813,327942,328035,329184,393326,393886,394256,394330,394380,394474,395089,395111,395954,395978,396190,396362,396931,396982,397373,398101,398787,399804,400754];+export default datacenterAsns;diff --git a/packages/geo/src/geo.ts b/packages/geo/src/geo.tsindex e2a5a148c..562a03ff0 100644--- a/packages/geo/src/geo.ts+++ b/packages/geo/src/geo.ts@@ -5,43 +5,52 @@ import { fileURLToPath } from 'node:url';import type { ReaderModel } from '@maxmind/geoip2-node';import { Reader } from '@maxmind/geoip2-node';import { LRUCache } from 'lru-cache';+import datacenterAsns from './datacenter-asns';const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);-const filename = 'GeoLite2-City.mmdb';-// From api or worker package-const dbPath = path.join(__dirname, `../../../packages/geo/${filename}`);-// From local package-const dbPathLocal = path.join(__dirname, `../${filename}`);--// Singleton promise - initialized once, awaited on every call-let readerPromise: Promise<ReaderModel | null> | null = null;--async function loadDatabase(): Promise<ReaderModel | null> {+// Resolve a bundled `.mmdb` file, trying the api/worker layout first and the+// local package layout second (mirrors how the file ships via `pnpm codegen`).+async function loadDatabase(filename: string): Promise<ReaderModel | null> {+ // From api or worker package+ const dbPath = path.join(__dirname, `../../../packages/geo/${filename}`);+ // From local package+ const dbPathLocal = path.join(__dirname, `../${filename}`);try {const dbBuffer = await readFile(dbPath);- console.log('GeoLite2-City.mmdb loaded (dist)', dbPath);+ console.log(`${filename} loaded (dist)`, dbPath);return Reader.openBuffer(dbBuffer);} catch {try {const dbBuffer = await readFile(dbPathLocal);- console.log('GeoLite2-City.mmdb loaded (local)', dbPathLocal);+ console.log(`${filename} loaded (local)`, dbPathLocal);return Reader.openBuffer(dbBuffer);} catch {- console.error('GeoLite2-City.mmdb not found', { dbPath, dbPathLocal });+ console.error(`${filename} not found`, { dbPath, dbPathLocal });return null;}}}+// Singleton promises - initialized once, awaited on every call+let readerPromise: Promise<ReaderModel | null> | null = null;+let asnReaderPromise: Promise<ReaderModel | null> | null = null;+function getReader(): Promise<ReaderModel | null> {if (!readerPromise) {- readerPromise = loadDatabase();+ readerPromise = loadDatabase('GeoLite2-City.mmdb');}return readerPromise;}+function getAsnReader(): Promise<ReaderModel | null> {+ if (!asnReaderPromise) {+ asnReaderPromise = loadDatabase('GeoLite2-ASN.mmdb');+ }+ return asnReaderPromise;+}+export interface GeoLocation {country: string | undefined;city: string | undefined;@@ -89,7 +98,64 @@ export async function getGeoLocation(ip?: string): Promise<GeoLocation> {};cache.set(ip, res);return res;- } catch (error) {+ } catch {+ // Cache negative lookups too — the reader throws AddressNotFoundError for+ // IPs absent from the db, and re-throwing on every event is wasted work.+ cache.set(ip, DEFAULT_GEO);return DEFAULT_GEO;}}++export interface AsnInfo {+ asn: number | undefined;+ org: string | undefined;+ // True when the ASN belongs to a datacenter / hosting provider (see+ // `datacenter-asns.ts`). Such traffic is unlikely to be a real end user.+ isHosting: boolean;+}++const DEFAULT_ASN: AsnInfo = {+ asn: undefined,+ org: undefined,+ isHosting: false,+};++const datacenterAsnSet = new Set<number>(datacenterAsns);++const asnCache = new LRUCache<string, AsnInfo>({+ max: 1000,+ ttl: 1000 * 60 * 5,+ ttlAutopurge: true,+});++// Resolve the Autonomous System an IP belongs to and whether it is a known+// datacenter / hosting network. Used to flag (not block) likely-bot traffic.+export async function getAsnInfo(ip?: string): Promise<AsnInfo> {+ if (!ip || ignore.includes(ip)) {+ return DEFAULT_ASN;+ }++ const cached = asnCache.get(ip);+ if (cached) {+ return cached;+ }++ const reader = await getAsnReader();++ try {+ const response = reader?.asn(ip);+ const asn = response?.autonomousSystemNumber;+ const res: AsnInfo = {+ asn,+ org: response?.autonomousSystemOrganization,+ isHosting: asn !== undefined && datacenterAsnSet.has(asn),+ };+ asnCache.set(ip, res);+ return res;+ } catch {+ // Cache negative lookups too — the reader throws AddressNotFoundError for+ // IPs absent from the db, and re-throwing on every event is wasted work.+ asnCache.set(ip, DEFAULT_ASN);+ return DEFAULT_ASN;+ }+}diff --git a/packages/geo/vitest.config.ts b/packages/geo/vitest.config.tsnew file mode 100644index 000000000..f87a2039f--- /dev/null+++ b/packages/geo/vitest.config.ts@@ -0,0 +1,3 @@+import { getSharedVitestConfig } from '../../vitest.shared';++export default getSharedVitestConfig({ __dirname });