Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Implement bot detection enhancements and database schema updates

Bill Yangb53679fb1db0
diff --git a/server/src/analytics-script/botSignals.test.ts b/server/src/analytics-script/botSignals.test.ts
index e9446ee5e..0ea7b3526 100644
--- a/server/src/analytics-script/botSignals.test.ts
+++ b/server/src/analytics-script/botSignals.test.ts
@@ -105,6 +105,31 @@ describe("getBotScore", () => {
expect(getBotSignalMask()).toBe(CLIENT_BOT_SIGNAL_MASKS.swiftShader);
});
+ it("ignores zero outer dimensions and skips the cache while prerendering", () => {
+ function setPrerendering(value: boolean) {
+ Object.defineProperty(document, "prerendering", {
+ value,
+ configurable: true,
+ });
+ }
+
+ setPrerendering(true);
+ setWindowProperty("outerHeight", 0);
+ setWindowProperty("outerWidth", 0);
+
+ // Prerendered pages legitimately report zero outer dimensions.
+ expect(getBotScore()).toBe(0);
+ expect(getBotSignalMask()).toBe(0);
+
+ // Nothing computed during prerender may stick: after activation the real
+ // environment is re-evaluated (and this time the zeros are suspicious).
+ setPrerendering(false);
+ expect(getBotScore()).toBe(2);
+ expect(getBotSignalMask()).toBe(CLIENT_BOT_SIGNAL_MASKS.zeroOuterDimensions);
+
+ setPrerendering(undefined as unknown as boolean);
+ });
+
it("caches the score for the page lifecycle", () => {
setNavigatorProperty("webdriver", true);
expect(getBotScore()).toBe(3);
diff --git a/server/src/analytics-script/botSignals.ts b/server/src/analytics-script/botSignals.ts
index 4ffe01a0f..0119fc336 100644
--- a/server/src/analytics-script/botSignals.ts
+++ b/server/src/analytics-script/botSignals.ts
@@ -36,7 +36,17 @@ export function getBotSignalMask(): number {
return getBotSignals().mask;
}
+function isPrerendering(): boolean {
+ return (document as { prerendering?: boolean }).prerendering === true;
+}
+
function getBotSignals(): BotSignalResult {
+ // A prerendered page reports zero outer dimensions and other non-representative
+ // state. Never cache a score computed before activation β€” recompute fresh so
+ // post-activation events see the real environment.
+ if (isPrerendering()) {
+ return calculateBotSignals();
+ }
cachedBotSignals ??= calculateBotSignals();
return cachedBotSignals;
}
@@ -89,8 +99,9 @@ function calculateBotSignals(): BotSignalResult {
addSignal(CLIENT_BOT_SIGNAL_MASKS.automationApi, 3);
}
- // 2. Zero outer dimensions β€” common in headless/browserless environments
- if (outerHeight === 0 || outerWidth === 0) {
+ // 2. Zero outer dimensions β€” common in headless/browserless environments.
+ // Skipped while prerendering: Chrome legitimately reports 0 there.
+ if ((outerHeight === 0 || outerWidth === 0) && !isPrerendering()) {
addSignal(CLIENT_BOT_SIGNAL_MASKS.zeroOuterDimensions, 2);
}
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.ts
index 9d2be85be..cd0421a34 100644
--- a/server/src/db/clickhouse/schema/core.ts
+++ b/server/src/db/clickhouse/schema/core.ts
@@ -22,6 +22,39 @@ const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
{ name: "is_datacenter_asn", definition: "is_datacenter_asn UInt8 DEFAULT 0" },
];
+const BOT_EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
+ // Prod tables created before session_id landed in the CREATE statement lack
+ // it; the insert already sends it, so healing the column starts populating it.
+ { name: "session_id", definition: "session_id String DEFAULT ''" },
+ // Null = the client sent no score and the server inferred nothing.
+ { name: "client_bot_score", definition: "client_bot_score Nullable(UInt8)" },
+ { name: "client_signal_mask", definition: "client_signal_mask UInt16 DEFAULT 0" },
+];
+
+async function ensureBotEventsColumns() {
+ const existingColumns = await getTableColumns("bot_events");
+ const missingColumns = BOT_EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));
+
+ if (missingColumns.length === 0) {
+ logger.debug("Bot events table columns are up to date");
+ return;
+ }
+
+ logger.info(
+ { missingColumns: missingColumns.map(column => column.name) },
+ "Adding missing bot events table columns"
+ );
+
+ await execClickhouseInitStep(
+ "add missing bot events columns",
+ `
+ ALTER TABLE bot_events
+ ${missingColumns.map(column => `ADD COLUMN IF NOT EXISTS ${column.definition}`).join(",\n ")}
+ `,
+ { lockAcquireTimeoutSeconds: 15 }
+ );
+}
+
async function ensureEventsColumns() {
const existingColumns = await getTableColumns("events");
const missingColumns = EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));
@@ -134,7 +167,9 @@ export async function initializeCoreTables() {
detected_bot_asn Bool DEFAULT false,
detected_rate_anomaly Bool DEFAULT false,
matched_ua_pattern String DEFAULT '',
- bot_category LowCardinality(String) DEFAULT ''
+ bot_category LowCardinality(String) DEFAULT '',
+ client_bot_score Nullable(UInt8),
+ client_signal_mask UInt16 DEFAULT 0
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
@@ -143,6 +178,8 @@ export async function initializeCoreTables() {
`
);
+ await ensureBotEventsColumns();
+
await execClickhouseInitStep(
"create session replay events table",
`
diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.ts
index 4fc7a2ccf..d04134a0f 100644
--- a/server/src/services/tracker/botBlocking/botEventQueue.ts
+++ b/server/src/services/tracker/botBlocking/botEventQueue.ts
@@ -75,6 +75,8 @@ class BotEventQueue {
detected_rate_anomaly: event.detectedRateAnomaly || false,
matched_ua_pattern: event.matchedUaPattern || "",
bot_category: event.botCategory || "",
+ client_bot_score: event.clientBotScore ?? null,
+ client_signal_mask: event.clientSignalMask ?? 0,
};
});
diff --git a/server/src/services/tracker/botBlocking/index.test.ts b/server/src/services/tracker/botBlocking/index.test.ts
index 6d71fe8ea..7b0b23e58 100644
--- a/server/src/services/tracker/botBlocking/index.test.ts
+++ b/server/src/services/tracker/botBlocking/index.test.ts
@@ -166,7 +166,12 @@ describe("checkBotBlocking", () => {
request: requestWithHeaders({ "user-agent": "okhttp/4.12.0" }),
blockBots: true,
isMobileSite: true,
- payload: { ...basePayload, userAgent: "okhttp/4.12.0", clientBotScore: 5 },
+ payload: {
+ ...basePayload,
+ userAgent: "okhttp/4.12.0",
+ clientBotScore: 3,
+ clientBotSignalMask: clientBotSignalMasks.automationApi,
+ },
});
expect(result).toMatchObject({
@@ -301,6 +306,85 @@ describe("checkBotBlocking", () => {
});
});
+ it("does not convict on weak client signals alone", async () => {
+ // zeroOuterDimensions (2) + swiftShader (1) reaches the score threshold,
+ // but both occur on real devices (prerendering, low-end GPUs) β€” weak
+ // signals corroborate, never convict.
+ const result = await checkBotBlocking({
+ request: requestWithHeaders(browserHeaders),
+ blockBots: true,
+ payload: {
+ ...basePayload,
+ clientBotScore: 3,
+ clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,
+ },
+ });
+
+ expect(result).toBeNull();
+ });
+
+ it("does not convict on a client score sent without a signal mask", async () => {
+ // Without the mask the score cannot be decomposed into strong vs. weak
+ // signals, so it can only ever corroborate.
+ const result = await checkBotBlocking({
+ request: requestWithHeaders(browserHeaders),
+ blockBots: true,
+ payload: { ...basePayload, clientBotScore: 5 },
+ });
+
+ expect(result).toBeNull();
+ });
+
+ it("uses weak client signals as supporting evidence when another layer matched", async () => {
+ const result = await checkBotBlocking({
+ request: requestWithHeaders({
+ ...browserHeaders,
+ "user-agent":
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36",
+ }),
+ blockBots: true,
+ payload: {
+ ...basePayload,
+ clientBotScore: 3,
+ clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,
+ },
+ });
+
+ expect(result).toMatchObject({
+ isBot: true,
+ message: "Bot detected using ua-pattern",
+ eventProperties: {
+ detectedUaPattern: true,
+ detectedClientSignals: true,
+ clientBotScore: 3,
+ clientSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,
+ },
+ });
+ expect(result?.detections.map(detection => detection.layer)).toContain("client_signals");
+ });
+
+ it("does not let two supporting signals convict each other", async () => {
+ // Weak client signals + generic hosting ASN are both supporting-only;
+ // together they still must not convict.
+ vi.mocked(lookupAsn).mockReturnValue({
+ asn: 16509,
+ organization: "Amazon.com, Inc.",
+ });
+
+ const result = await checkBotBlocking({
+ request: requestWithHeaders(browserHeaders),
+ blockBots: true,
+ payload: {
+ ...basePayload,
+ ipAddress: "18.0.0.1",
+ clientBotScore: 3,
+ clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,
+ },
+ });
+
+ expect(result).toBeNull();
+ });
+
it("does not block generic hosting ASN by itself", async () => {
vi.mocked(lookupAsn).mockReturnValue({
asn: 16509,
diff --git a/server/src/services/tracker/botBlocking/index.ts b/server/src/services/tracker/botBlocking/index.ts
index f409d898f..5cc1434b3 100644
--- a/server/src/services/tracker/botBlocking/index.ts
+++ b/server/src/services/tracker/botBlocking/index.ts
@@ -62,6 +62,40 @@ const CLIENT_SIGNAL_MASKS = {
type ClientSignalName = keyof typeof CLIENT_SIGNAL_MASKS;
+// Must mirror the weights in analytics-script/botSignals.ts β€” the client sends
+// one cached score, and the server recomputes it from the signal mask to split
+// it into convicting vs. corroborating evidence.
+const CLIENT_SIGNAL_WEIGHTS: Record<ClientSignalName, number> = {
+ automationApi: 3,
+ zeroOuterDimensions: 2,
+ missingChrome: 1,
+ swiftShader: 1,
+ emptyPlugins: 1,
+ defaultViewport800x600: 3,
+ defaultViewport1024x768: 3,
+ impossibleDimensions: 3,
+ outerDimensionsWeird: 2,
+ pluginApiAbsence: 0,
+};
+
+// Signals that are automation-specific enough to convict on their own. The
+// remaining (weak) signals β€” empty plugins, SwiftShader, zero outer dimensions,
+// missing window.chrome β€” all occur on real devices (Android Chrome ships empty
+// plugins; low-end GPUs fall back to SwiftShader; prerendered pages report zero
+// outer dimensions), so they corroborate other layers but never convict.
+const STRONG_CLIENT_SIGNAL_BITS =
+ CLIENT_SIGNAL_MASKS.automationApi |
+ CLIENT_SIGNAL_MASKS.impossibleDimensions |
+ CLIENT_SIGNAL_MASKS.defaultViewport800x600 |
+ CLIENT_SIGNAL_MASKS.defaultViewport1024x768;
+
+function sumClientSignalWeights(mask: number): number {
+ return Object.entries(CLIENT_SIGNAL_MASKS).reduce(
+ (total, [name, bit]) => ((mask & bit) !== 0 ? total + CLIENT_SIGNAL_WEIGHTS[name as ClientSignalName] : total),
+ 0
+ );
+}
+
export interface BotBlockingDetection {
layer: BotDetectionMethod;
botCategory?: string | null;
@@ -92,6 +126,8 @@ export interface BotEventProperties {
detectedRateAnomaly: boolean;
matchedUaPattern: string;
botCategory: string;
+ clientBotScore?: number;
+ clientSignalMask?: number;
}
export interface BotDetectionResult {
@@ -101,7 +137,11 @@ export interface BotDetectionResult {
eventProperties: BotEventProperties;
}
-function buildBotEventProperties(detections: BotBlockingDetection[], asnInfo: AsnInfo | null): BotEventProperties {
+function buildBotEventProperties(
+ detections: BotBlockingDetection[],
+ asnInfo: AsnInfo | null,
+ clientSignalResult: ReturnType<typeof getClientSignalResult>
+): BotEventProperties {
const detectionLayers = new Set(detections.map(detection => detection.layer));
const uaDetection = detections.find(detection => detection.layer === "ua_pattern");
@@ -116,6 +156,8 @@ function buildBotEventProperties(detections: BotBlockingDetection[], asnInfo: As
detectedRateAnomaly: detectionLayers.has("rate_anomaly"),
matchedUaPattern: uaDetection?.matchedPattern ?? "",
botCategory: uaDetection?.botCategory ?? "",
+ clientBotScore: clientSignalResult.scoreForStats,
+ clientSignalMask: clientSignalResult.maskForStats,
};
}
@@ -174,8 +216,14 @@ function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {
const score = Math.min((hasClientScore ? payload.clientBotScore! : 0) + inferredScore, 10);
+ // Score contributed by strong (convicting) signals only. Derived from the
+ // mask, not the client's opaque score β€” a score sent without a mask cannot be
+ // decomposed, so it can only ever corroborate.
+ const strongScore = sumClientSignalWeights(mask & STRONG_CLIENT_SIGNAL_BITS);
+
return {
score,
+ strongScore,
mask,
signalNames: getClientSignalNames(mask),
scoreForStats: hasClientScore || inferredScore > 0 ? score : undefined,
@@ -234,14 +282,24 @@ export async function checkBotBlocking({
}
}
- // Layer 3: Client-side and client-derived bot signal score check
- if (clientSignalResult.score >= CLIENT_BOT_SCORE_THRESHOLD) {
- addDetection("Bot detected using client signals", {
+ // Layer 3: Client-side and client-derived bot signal score check. Only
+ // strong signals convict on their own; weak signals reaching the threshold
+ // (or a score the client sent without a mask) corroborate other layers but
+ // never convict, mirroring the generic-hosting-ASN rule below.
+ let supportingClientSignalDetection: BotBlockingDetection | null = null;
+ if (Math.max(clientSignalResult.score, clientSignalResult.strongScore) >= CLIENT_BOT_SCORE_THRESHOLD) {
+ const clientSignalDetection: BotBlockingDetection = {
layer: "client_signals",
clientBotScore: clientSignalResult.score,
clientBotSignalMask: clientSignalResult.mask,
clientSignals: clientSignalResult.signalNames,
- });
+ };
+
+ if (clientSignalResult.strongScore >= CLIENT_BOT_SCORE_THRESHOLD) {
+ addDetection("Bot detected using client signals", clientSignalDetection);
+ } else {
+ supportingClientSignalDetection = clientSignalDetection;
+ }
}
// Layer 4: ASN check β€” IP belongs to hosting/cloud or curated bot provider infrastructure.
@@ -289,7 +347,13 @@ export async function checkBotBlocking({
});
}
- if (supportingHostingAsnDetection && detections.length > 0) {
+ // Supporting evidence attaches only when a convicting layer already fired β€”
+ // two supporting signals must not convict each other.
+ const hasConvictingDetection = detections.length > 0;
+ if (hasConvictingDetection && supportingClientSignalDetection) {
+ addDetection("Bot detected using client signals", supportingClientSignalDetection);
+ }
+ if (hasConvictingDetection && supportingHostingAsnDetection) {
addDetection("Bot detected using bot asn", supportingHostingAsnDetection);
}
@@ -315,6 +379,6 @@ export async function checkBotBlocking({
isBot: true,
message: blockMessage ?? "Bot detected",
detections,
- eventProperties: buildBotEventProperties(detections, asnInfo),
+ eventProperties: buildBotEventProperties(detections, asnInfo, clientSignalResult),
};
}