πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Implement bot detection enhancements and database schema updates
Bill Yangb53679fb1db0
diff --git a/server/src/analytics-script/botSignals.test.ts b/server/src/analytics-script/botSignals.test.tsindex e9446ee5e..0ea7b3526 100644--- a/server/src/analytics-script/botSignals.test.ts+++ b/server/src/analytics-script/botSignals.test.ts@@ -105,6 +105,31 @@ describe("getBotScore", () => {expect(getBotSignalMask()).toBe(CLIENT_BOT_SIGNAL_MASKS.swiftShader);});+ it("ignores zero outer dimensions and skips the cache while prerendering", () => {+ function setPrerendering(value: boolean) {+ Object.defineProperty(document, "prerendering", {+ value,+ configurable: true,+ });+ }++ setPrerendering(true);+ setWindowProperty("outerHeight", 0);+ setWindowProperty("outerWidth", 0);++ // Prerendered pages legitimately report zero outer dimensions.+ expect(getBotScore()).toBe(0);+ expect(getBotSignalMask()).toBe(0);++ // Nothing computed during prerender may stick: after activation the real+ // environment is re-evaluated (and this time the zeros are suspicious).+ setPrerendering(false);+ expect(getBotScore()).toBe(2);+ expect(getBotSignalMask()).toBe(CLIENT_BOT_SIGNAL_MASKS.zeroOuterDimensions);++ setPrerendering(undefined as unknown as boolean);+ });+it("caches the score for the page lifecycle", () => {setNavigatorProperty("webdriver", true);expect(getBotScore()).toBe(3);diff --git a/server/src/analytics-script/botSignals.ts b/server/src/analytics-script/botSignals.tsindex 4ffe01a0f..0119fc336 100644--- a/server/src/analytics-script/botSignals.ts+++ b/server/src/analytics-script/botSignals.ts@@ -36,7 +36,17 @@ export function getBotSignalMask(): number {return getBotSignals().mask;}+function isPrerendering(): boolean {+ return (document as { prerendering?: boolean }).prerendering === true;+}+function getBotSignals(): BotSignalResult {+ // A prerendered page reports zero outer dimensions and other non-representative+ // state. Never cache a score computed before activation β recompute fresh so+ // post-activation events see the real environment.+ if (isPrerendering()) {+ return calculateBotSignals();+ }cachedBotSignals ??= calculateBotSignals();return cachedBotSignals;}@@ -89,8 +99,9 @@ function calculateBotSignals(): BotSignalResult {addSignal(CLIENT_BOT_SIGNAL_MASKS.automationApi, 3);}- // 2. Zero outer dimensions β common in headless/browserless environments- if (outerHeight === 0 || outerWidth === 0) {+ // 2. Zero outer dimensions β common in headless/browserless environments.+ // Skipped while prerendering: Chrome legitimately reports 0 there.+ if ((outerHeight === 0 || outerWidth === 0) && !isPrerendering()) {addSignal(CLIENT_BOT_SIGNAL_MASKS.zeroOuterDimensions, 2);}diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.tsindex 9d2be85be..cd0421a34 100644--- a/server/src/db/clickhouse/schema/core.ts+++ b/server/src/db/clickhouse/schema/core.ts@@ -22,6 +22,39 @@ const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [{ name: "is_datacenter_asn", definition: "is_datacenter_asn UInt8 DEFAULT 0" },];+const BOT_EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [+ // Prod tables created before session_id landed in the CREATE statement lack+ // it; the insert already sends it, so healing the column starts populating it.+ { name: "session_id", definition: "session_id String DEFAULT ''" },+ // Null = the client sent no score and the server inferred nothing.+ { name: "client_bot_score", definition: "client_bot_score Nullable(UInt8)" },+ { name: "client_signal_mask", definition: "client_signal_mask UInt16 DEFAULT 0" },+];++async function ensureBotEventsColumns() {+ const existingColumns = await getTableColumns("bot_events");+ const missingColumns = BOT_EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));++ if (missingColumns.length === 0) {+ logger.debug("Bot events table columns are up to date");+ return;+ }++ logger.info(+ { missingColumns: missingColumns.map(column => column.name) },+ "Adding missing bot events table columns"+ );++ await execClickhouseInitStep(+ "add missing bot events columns",+ `+ ALTER TABLE bot_events+ ${missingColumns.map(column => `ADD COLUMN IF NOT EXISTS ${column.definition}`).join(",\n ")}+ `,+ { lockAcquireTimeoutSeconds: 15 }+ );+}+async function ensureEventsColumns() {const existingColumns = await getTableColumns("events");const missingColumns = EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));@@ -134,7 +167,9 @@ export async function initializeCoreTables() {detected_bot_asn Bool DEFAULT false,detected_rate_anomaly Bool DEFAULT false,matched_ua_pattern String DEFAULT '',- bot_category LowCardinality(String) DEFAULT ''+ bot_category LowCardinality(String) DEFAULT '',+ client_bot_score Nullable(UInt8),+ client_signal_mask UInt16 DEFAULT 0)ENGINE = MergeTree()PARTITION BY toYYYYMM(timestamp)@@ -143,6 +178,8 @@ export async function initializeCoreTables() {`);+ await ensureBotEventsColumns();+await execClickhouseInitStep("create session replay events table",`diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.tsindex 4fc7a2ccf..d04134a0f 100644--- a/server/src/services/tracker/botBlocking/botEventQueue.ts+++ b/server/src/services/tracker/botBlocking/botEventQueue.ts@@ -75,6 +75,8 @@ class BotEventQueue {detected_rate_anomaly: event.detectedRateAnomaly || false,matched_ua_pattern: event.matchedUaPattern || "",bot_category: event.botCategory || "",+ client_bot_score: event.clientBotScore ?? null,+ client_signal_mask: event.clientSignalMask ?? 0,};});diff --git a/server/src/services/tracker/botBlocking/index.test.ts b/server/src/services/tracker/botBlocking/index.test.tsindex 6d71fe8ea..7b0b23e58 100644--- a/server/src/services/tracker/botBlocking/index.test.ts+++ b/server/src/services/tracker/botBlocking/index.test.ts@@ -166,7 +166,12 @@ describe("checkBotBlocking", () => {request: requestWithHeaders({ "user-agent": "okhttp/4.12.0" }),blockBots: true,isMobileSite: true,- payload: { ...basePayload, userAgent: "okhttp/4.12.0", clientBotScore: 5 },+ payload: {+ ...basePayload,+ userAgent: "okhttp/4.12.0",+ clientBotScore: 3,+ clientBotSignalMask: clientBotSignalMasks.automationApi,+ },});expect(result).toMatchObject({@@ -301,6 +306,85 @@ describe("checkBotBlocking", () => {});});+ it("does not convict on weak client signals alone", async () => {+ // zeroOuterDimensions (2) + swiftShader (1) reaches the score threshold,+ // but both occur on real devices (prerendering, low-end GPUs) β weak+ // signals corroborate, never convict.+ const result = await checkBotBlocking({+ request: requestWithHeaders(browserHeaders),+ blockBots: true,+ payload: {+ ...basePayload,+ clientBotScore: 3,+ clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,+ },+ });++ expect(result).toBeNull();+ });++ it("does not convict on a client score sent without a signal mask", async () => {+ // Without the mask the score cannot be decomposed into strong vs. weak+ // signals, so it can only ever corroborate.+ const result = await checkBotBlocking({+ request: requestWithHeaders(browserHeaders),+ blockBots: true,+ payload: { ...basePayload, clientBotScore: 5 },+ });++ expect(result).toBeNull();+ });++ it("uses weak client signals as supporting evidence when another layer matched", async () => {+ const result = await checkBotBlocking({+ request: requestWithHeaders({+ ...browserHeaders,+ "user-agent":+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36",+ }),+ blockBots: true,+ payload: {+ ...basePayload,+ clientBotScore: 3,+ clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,+ },+ });++ expect(result).toMatchObject({+ isBot: true,+ message: "Bot detected using ua-pattern",+ eventProperties: {+ detectedUaPattern: true,+ detectedClientSignals: true,+ clientBotScore: 3,+ clientSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,+ },+ });+ expect(result?.detections.map(detection => detection.layer)).toContain("client_signals");+ });++ it("does not let two supporting signals convict each other", async () => {+ // Weak client signals + generic hosting ASN are both supporting-only;+ // together they still must not convict.+ vi.mocked(lookupAsn).mockReturnValue({+ asn: 16509,+ organization: "Amazon.com, Inc.",+ });++ const result = await checkBotBlocking({+ request: requestWithHeaders(browserHeaders),+ blockBots: true,+ payload: {+ ...basePayload,+ ipAddress: "18.0.0.1",+ clientBotScore: 3,+ clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,+ },+ });++ expect(result).toBeNull();+ });+it("does not block generic hosting ASN by itself", async () => {vi.mocked(lookupAsn).mockReturnValue({asn: 16509,diff --git a/server/src/services/tracker/botBlocking/index.ts b/server/src/services/tracker/botBlocking/index.tsindex f409d898f..5cc1434b3 100644--- a/server/src/services/tracker/botBlocking/index.ts+++ b/server/src/services/tracker/botBlocking/index.ts@@ -62,6 +62,40 @@ const CLIENT_SIGNAL_MASKS = {type ClientSignalName = keyof typeof CLIENT_SIGNAL_MASKS;+// Must mirror the weights in analytics-script/botSignals.ts β the client sends+// one cached score, and the server recomputes it from the signal mask to split+// it into convicting vs. corroborating evidence.+const CLIENT_SIGNAL_WEIGHTS: Record<ClientSignalName, number> = {+ automationApi: 3,+ zeroOuterDimensions: 2,+ missingChrome: 1,+ swiftShader: 1,+ emptyPlugins: 1,+ defaultViewport800x600: 3,+ defaultViewport1024x768: 3,+ impossibleDimensions: 3,+ outerDimensionsWeird: 2,+ pluginApiAbsence: 0,+};++// Signals that are automation-specific enough to convict on their own. The+// remaining (weak) signals β empty plugins, SwiftShader, zero outer dimensions,+// missing window.chrome β all occur on real devices (Android Chrome ships empty+// plugins; low-end GPUs fall back to SwiftShader; prerendered pages report zero+// outer dimensions), so they corroborate other layers but never convict.+const STRONG_CLIENT_SIGNAL_BITS =+ CLIENT_SIGNAL_MASKS.automationApi |+ CLIENT_SIGNAL_MASKS.impossibleDimensions |+ CLIENT_SIGNAL_MASKS.defaultViewport800x600 |+ CLIENT_SIGNAL_MASKS.defaultViewport1024x768;++function sumClientSignalWeights(mask: number): number {+ return Object.entries(CLIENT_SIGNAL_MASKS).reduce(+ (total, [name, bit]) => ((mask & bit) !== 0 ? total + CLIENT_SIGNAL_WEIGHTS[name as ClientSignalName] : total),+ 0+ );+}+export interface BotBlockingDetection {layer: BotDetectionMethod;botCategory?: string | null;@@ -92,6 +126,8 @@ export interface BotEventProperties {detectedRateAnomaly: boolean;matchedUaPattern: string;botCategory: string;+ clientBotScore?: number;+ clientSignalMask?: number;}export interface BotDetectionResult {@@ -101,7 +137,11 @@ export interface BotDetectionResult {eventProperties: BotEventProperties;}-function buildBotEventProperties(detections: BotBlockingDetection[], asnInfo: AsnInfo | null): BotEventProperties {+function buildBotEventProperties(+ detections: BotBlockingDetection[],+ asnInfo: AsnInfo | null,+ clientSignalResult: ReturnType<typeof getClientSignalResult>+): BotEventProperties {const detectionLayers = new Set(detections.map(detection => detection.layer));const uaDetection = detections.find(detection => detection.layer === "ua_pattern");@@ -116,6 +156,8 @@ function buildBotEventProperties(detections: BotBlockingDetection[], asnInfo: AsdetectedRateAnomaly: detectionLayers.has("rate_anomaly"),matchedUaPattern: uaDetection?.matchedPattern ?? "",botCategory: uaDetection?.botCategory ?? "",+ clientBotScore: clientSignalResult.scoreForStats,+ clientSignalMask: clientSignalResult.maskForStats,};}@@ -174,8 +216,14 @@ function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {const score = Math.min((hasClientScore ? payload.clientBotScore! : 0) + inferredScore, 10);+ // Score contributed by strong (convicting) signals only. Derived from the+ // mask, not the client's opaque score β a score sent without a mask cannot be+ // decomposed, so it can only ever corroborate.+ const strongScore = sumClientSignalWeights(mask & STRONG_CLIENT_SIGNAL_BITS);+return {score,+ strongScore,mask,signalNames: getClientSignalNames(mask),scoreForStats: hasClientScore || inferredScore > 0 ? score : undefined,@@ -234,14 +282,24 @@ export async function checkBotBlocking({}}- // Layer 3: Client-side and client-derived bot signal score check- if (clientSignalResult.score >= CLIENT_BOT_SCORE_THRESHOLD) {- addDetection("Bot detected using client signals", {+ // Layer 3: Client-side and client-derived bot signal score check. Only+ // strong signals convict on their own; weak signals reaching the threshold+ // (or a score the client sent without a mask) corroborate other layers but+ // never convict, mirroring the generic-hosting-ASN rule below.+ let supportingClientSignalDetection: BotBlockingDetection | null = null;+ if (Math.max(clientSignalResult.score, clientSignalResult.strongScore) >= CLIENT_BOT_SCORE_THRESHOLD) {+ const clientSignalDetection: BotBlockingDetection = {layer: "client_signals",clientBotScore: clientSignalResult.score,clientBotSignalMask: clientSignalResult.mask,clientSignals: clientSignalResult.signalNames,- });+ };++ if (clientSignalResult.strongScore >= CLIENT_BOT_SCORE_THRESHOLD) {+ addDetection("Bot detected using client signals", clientSignalDetection);+ } else {+ supportingClientSignalDetection = clientSignalDetection;+ }}// Layer 4: ASN check β IP belongs to hosting/cloud or curated bot provider infrastructure.@@ -289,7 +347,13 @@ export async function checkBotBlocking({});}- if (supportingHostingAsnDetection && detections.length > 0) {+ // Supporting evidence attaches only when a convicting layer already fired β+ // two supporting signals must not convict each other.+ const hasConvictingDetection = detections.length > 0;+ if (hasConvictingDetection && supportingClientSignalDetection) {+ addDetection("Bot detected using client signals", supportingClientSignalDetection);+ }+ if (hasConvictingDetection && supportingHostingAsnDetection) {addDetection("Bot detected using bot asn", supportingHostingAsnDetection);}@@ -315,6 +379,6 @@ export async function checkBotBlocking({isBot: true,message: blockMessage ?? "Bot detected",detections,- eventProperties: buildBotEventProperties(detections, asnInfo),+ eventProperties: buildBotEventProperties(detections, asnInfo, clientSignalResult),};}