πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
massive perf improvements (#1017)
Goldflagad63515e0ca1
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.ymlindex b6a21d1e9..71b342545 100644--- a/docker-compose.cloud.yml+++ b/docker-compose.cloud.yml@@ -73,12 +73,29 @@ services:redis:image: redis:7-alpinecontainer_name: redis- ports:- - "6379:6379" # Exposed to internet for remote connectionsvolumes:- redis-data:/data+ healthcheck:+ test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD:-changeme}", "--no-auth-warning", "ping"]+ interval: 3s+ timeout: 5s+ retries: 5+ start_period: 5srestart: unless-stopped- command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:-changeme}", "--appendonly", "yes"]+ # Backs session tracking and the BullMQ queues. noeviction is required by+ # BullMQ (evicting job keys corrupts queues); session keys carry a 30-minute+ # TTL so memory stays bounded by active-user count. AOF (everysec) keeps+ # queues durable across restarts.+ command:+ - redis-server+ - --requirepass+ - ${REDIS_PASSWORD:-changeme}+ - --appendonly+ - "yes"+ - --appendfsync+ - everysec+ - --maxmemory-policy+ - noevictionbackend:container_name: backend@@ -134,6 +151,8 @@ services:condition: service_healthypostgres:condition: service_started+ redis:+ condition: service_healthyrestart: unless-stoppedports:- "3001:3001"diff --git a/docker-compose.yml b/docker-compose.ymlindex 114168e3f..ffafbae80 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -65,6 +65,33 @@ services:ports:- "5432:5432"+ redis:+ image: redis:7-alpine+ container_name: redis+ volumes:+ - redis-data:/data+ healthcheck:+ test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD:-changeme}", "--no-auth-warning", "ping"]+ interval: 3s+ timeout: 5s+ retries: 5+ start_period: 5s+ restart: unless-stopped+ # Backs session tracking and the BullMQ queues. noeviction is required by+ # BullMQ (evicting job keys corrupts queues); session keys carry a 30-minute+ # TTL so memory stays bounded by active-user count. AOF (everysec) keeps+ # queues durable across restarts.+ command:+ - redis-server+ - --requirepass+ - ${REDIS_PASSWORD:-changeme}+ - --appendonly+ - "yes"+ - --appendfsync+ - everysec+ - --maxmemory-policy+ - noeviction+backend:image: ghcr.io/rybbit-io/rybbit-backend:${IMAGE_TAG:-latest}container_name: backend@@ -83,6 +110,9 @@ services:- POSTGRES_DB=${POSTGRES_DB:-analytics}- POSTGRES_USER=${POSTGRES_USER:-frog}- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-frog}+ - REDIS_HOST=redis+ - REDIS_PORT=6379+ - REDIS_PASSWORD=${REDIS_PASSWORD:-changeme}- BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET}- BASE_URL=${BASE_URL}- DISABLE_SIGNUP=${DISABLE_SIGNUP}@@ -98,6 +128,8 @@ services:condition: service_healthypostgres:condition: service_started+ redis:+ condition: service_healthyhealthcheck:test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3001/api/health"]interval: 3s@@ -131,6 +163,7 @@ services:volumes:clickhouse-data:postgres-data:+ redis-data:caddy_data: # Persistent volume for Caddy's certificates and statecaddy_config: # Persistent volume for Caddy's configuration cache (optional but good practice)diff --git a/server/.env.example b/server/.env.exampleindex 6e94203e1..25db15c51 100644--- a/server/.env.example+++ b/server/.env.example@@ -5,8 +5,6 @@ CLICKHOUSE_HOST=http://localhost:8123CLICKHOUSE_DB=analyticsCLICKHOUSE_USER=defaultCLICKHOUSE_PASSWORD=-CLICKHOUSE_READONLY_USER=readonly-CLICKHOUSE_READONLY_PASSWORD=# PostgreSQL configurationPOSTGRES_HOST=localhost@@ -15,10 +13,10 @@ POSTGRES_DB=analyticsPOSTGRES_USER=postgresPOSTGRES_PASSWORD=-# Redis configuration (for BullMQ)+# Redis configurationREDIS_HOST=localhostREDIS_PORT=6379-REDIS_PASSWORD=your-secure-password-here+REDIS_PASSWORD=# AuthenticationBETTER_AUTH_SECRET=your-secret-key-herediff --git a/server/package-lock.json b/server/package-lock.jsonindex de1d47d3b..92270af44 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -29,6 +29,7 @@"drizzle-orm": "0.45.1","fastify": "5.8.5","i18n-iso-countries": "^7.14.0",+ "ioredis": "^5.8.2","ip-address": "^10.0.1","luxon": "3.7.2","nanoid": "^5.1.6",diff --git a/server/package.json b/server/package.jsonindex fc6535f54..56ab00c65 100644--- a/server/package.json+++ b/server/package.json@@ -49,6 +49,7 @@"drizzle-orm": "0.45.1","fastify": "5.8.5","i18n-iso-countries": "^7.14.0",+ "ioredis": "^5.8.2","ip-address": "^10.0.1","luxon": "3.7.2","nanoid": "^5.1.6",diff --git a/server/src/cluster.ts b/server/src/cluster.tsindex d1c5d949f..24cab5313 100644--- a/server/src/cluster.ts+++ b/server/src/cluster.ts@@ -47,7 +47,6 @@ if (workerCount === 0) {// Start cron jobs on the primary process onlytelemetryService.startTelemetryCron();- sessionsService.startCleanupCron();usageService.startUsageCheckCron();if (IS_CLOUD && process.env.NODE_ENV !== "development") {weeklyReportService.startWeeklyReportCron();@@ -117,7 +116,7 @@ if (workerCount === 0) {// Stop cron jobsusageService.stopUsageCheckCron();- sessionsService.stopCleanupCron();+ void sessionsService.close();telemetryService.stopTelemetryCron();if (IS_CLOUD) {weeklyReportService.stopWeeklyReportCron();diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex 2a81c5d1a..d82812de5 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -2,12 +2,7 @@ import { createClient } from "@clickhouse/client";import { IS_CLOUD, LITE_DASHBOARD } from "../../lib/const.js";import { createServiceLogger } from "../../lib/logger/logger.js";-const parsePositiveInt = (value: string | undefined, fallback: number) => {- const parsed = Number(value);- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;-};--const CLICKHOUSE_REQUEST_TIMEOUT_MS = parsePositiveInt(process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS, 300_000);+const CLICKHOUSE_REQUEST_TIMEOUT_MS = 300_000;export const clickhouse = createClient({url: process.env.CLICKHOUSE_HOST,diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 49883d0d2..30f0f51cb 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -95,7 +95,11 @@ export const sites = pgTable(table => [check("sites_type_check", sql`${table.type} IS NULL OR ${table.type} IN ('web', 'mobile')`)]);-// Active sessions table+// Active sessions table.+// DEPRECATED: session tracking moved to Redis (see services/sessions/sessionsService.ts).+// No longer read or written by the app; kept so existing deployments stay drift-free.+// Drop it once Redis-backed sessions are verified in production:+// DROP TABLE IF EXISTS active_sessions;export const activeSessions = pgTable("active_sessions", {sessionId: text("session_id").primaryKey().notNull(),siteId: integer("site_id"),diff --git a/server/src/db/redis/redis.ts b/server/src/db/redis/redis.tsnew file mode 100644index 000000000..0258acef0--- /dev/null+++ b/server/src/db/redis/redis.ts@@ -0,0 +1,117 @@+import { Redis } from "ioredis";+import { createServiceLogger } from "../../lib/logger/logger.js";++const logger = createServiceLogger("redis");++// Fail fast so callers can fall back rather than hanging during an outage.+const REDIS_COMMAND_TIMEOUT_MS = 1000;++// A request/response Redis client shared across the process. Distinct from the+// BullMQ connections used by the uptime service (those have their own settings).+export const redis = new Redis({+ host: process.env.REDIS_HOST || "localhost",+ port: parseInt(process.env.REDIS_PORT || "6379", 10),+ ...(process.env.REDIS_PASSWORD ? { password: process.env.REDIS_PASSWORD } : {}),+ commandTimeout: REDIS_COMMAND_TIMEOUT_MS,+ maxRetriesPerRequest: 3,+ retryStrategy: (times: number) => Math.min(times * 200, 5000),+});++redis.on("error", (error: Error) => logger.error(error, "Redis client error"));+redis.on("connect", () => logger.info("Redis connected"));++// Atomic get-or-create with a sliding TTL. If the key exists we refresh its TTL+// and return the stored id; otherwise we store the candidate and return it. Run+// as a single Lua script so concurrent workers can never create duplicate+// sessions for the same (siteId, userId). defineCommand uses EVALSHA with an+// EVAL fallback, so the script body is only sent over the wire once.+redis.defineCommand("sessionGetOrCreate", {+ numberOfKeys: 1,+ lua: `+ local existing = redis.call('GET', KEYS[1])+ if existing then+ redis.call('PEXPIRE', KEYS[1], ARGV[2])+ return existing+ end+ redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])+ return ARGV[1]+ `,+});++interface SessionRedis extends Redis {+ sessionGetOrCreate(key: string, candidateId: string, ttlMs: number): Promise<string>;+}++/**+ * Return the session id for `key`, creating it from `candidateId` if absent, and+ * (re)setting its TTL to `ttlMs`. Atomic across all workers.+ */+export function sessionGetOrCreate(key: string, candidateId: string, ttlMs: number): Promise<string> {+ return (redis as SessionRedis).sessionGetOrCreate(key, candidateId, ttlMs);+}++// Rolling-window counters for the bot anomaly scorer. Each counter is a sorted+// set: member -> observation, score -> time (ms). For event-rate counters the+// member is a unique per-event token (so cardinality == event count); for+// distinct counters the member is the observed value (so cardinality == distinct+// count, and re-observing refreshes its score). For every counter we add the+// observation, drop anything older than its window, cap the set size, refresh the+// key TTL, and read back the cardinality.+//+// All of a request's counters run in one Lua call so anomaly scoring costs a+// single round-trip, and because the state lives in Redis every worker/replica+// shares one view β an in-process Map would only ever see 1/N of the traffic+// behind a load balancer, diluting the thresholds by the worker count.+//+// numberOfKeys is omitted so the key count can be passed per call. KEYS holds the+// counter keys; ARGV is [nowMs, member1, windowMs1, maxSize1, member2, ...] with+// one (member, windowMs, maxSize) triple per key. Returns one count per key.+redis.defineCommand("anomalyObserve", {+ lua: `+ local now = tonumber(ARGV[1])+ local results = {}+ for i = 1, #KEYS do+ local key = KEYS[i]+ local base = (i - 1) * 3 + 1+ local member = ARGV[base + 1]+ local windowMs = tonumber(ARGV[base + 2])+ local maxSize = tonumber(ARGV[base + 3])+ redis.call('ZADD', key, now, member)+ redis.call('ZREMRANGEBYSCORE', key, '-inf', '(' .. (now - windowMs))+ if maxSize > 0 then+ redis.call('ZREMRANGEBYRANK', key, 0, -maxSize - 1)+ end+ redis.call('PEXPIRE', key, windowMs)+ results[i] = redis.call('ZCARD', key)+ end+ return results+ `,+});++export interface AnomalyCounterSpec {+ /** Fully-namespaced Redis key for this counter. */+ key: string;+ /** Unique per-event token (rate counters) or observed value (distinct counters). */+ member: string;+ windowMs: number;+ /** Hard cap on stored members; 0 disables trimming. */+ maxSize: number;+}++interface AnomalyRedis extends Redis {+ anomalyObserve(...args: (string | number)[]): Promise<number[]>;+}++/**+ * Observe one event against a batch of rolling-window counters and return each+ * counter's current cardinality, in the same order as `specs`. Atomic across all+ * workers; a single round-trip regardless of how many counters are passed.+ */+export function anomalyObserve(nowMs: number, specs: AnomalyCounterSpec[]): Promise<number[]> {+ const keys = specs.map(spec => spec.key);+ const args: (string | number)[] = [specs.length, ...keys, nowMs];+ for (const spec of specs) {+ args.push(spec.member, spec.windowMs, spec.maxSize);+ }+ return (redis as AnomalyRedis).anomalyObserve(...args);+}diff --git a/server/src/index.ts b/server/src/index.tsindex 40b04385f..2a85fc4c5 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -162,7 +162,6 @@ import { auth } from "./lib/auth.js";import { createCorsOptionsDelegate, createRejectUntrustedOriginHook } from "./lib/cors.js";import { IS_CLOUD } from "./lib/const.js";import { reengagementService } from "./services/reengagement/reengagementService.js";-import { sessionsService } from "./services/sessions/sessionsService.js";import { telemetryService } from "./services/telemetryService.js";import { handleIdentify } from "./services/tracker/identifyService.js";import { trackEvent } from "./services/tracker/trackEvent.js";@@ -256,10 +255,14 @@ server.register({ auth: auth! });-// Serve analytics scripts with generic names to avoid ad-blocker detection-server.get("/api/script.js", async (_, reply) => reply.sendFile("script.js"));-server.get("/api/replay.js", async (_, reply) => reply.sendFile("rrweb.min.js"));-server.get("/api/metrics.js", async (_, reply) => reply.sendFile("web-vitals.iife.js"));+// Serve analytics scripts with generic names to avoid ad-blocker detection.+// Cache them so browsers stop revalidating on every page load β without this they+// default to max-age=0, so each tracked page hit fires a conditional request that+// lands on caddy and the backend. script.js gets a short TTL so tracker updates+// still propagate quickly; the vendored libs rarely change and get a longer TTL.+server.get("/api/script.js", async (_, reply) => reply.sendFile("script.js", { maxAge: "1h" }));+server.get("/api/replay.js", async (_, reply) => reply.sendFile("rrweb.min.js", { maxAge: "1d" }));+server.get("/api/metrics.js", async (_, reply) => reply.sendFile("web-vitals.iife.js", { maxAge: "1d" }));// Domain-specific route pluginsasync function analyticsRoutes(fastify: FastifyInstance) {@@ -472,7 +475,6 @@ const start = async () => {// Cron jobs should only run on the primary process (or in single-process mode)if (!cluster.isWorker) {telemetryService.startTelemetryCron();- sessionsService.startCleanupCron();usageService.startUsageCheckCron();if (IS_CLOUD && process.env.NODE_ENV !== "development") {weeklyReportService.startWeeklyReportCron();diff --git a/server/src/services/sessions/sessionsService.test.ts b/server/src/services/sessions/sessionsService.test.tsnew file mode 100644index 000000000..6ee5344a6--- /dev/null+++ b/server/src/services/sessions/sessionsService.test.ts@@ -0,0 +1,85 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ sessionGetOrCreate: vi.fn(),+ quit: vi.fn(),+}));++vi.mock("../../db/redis/redis.js", () => ({+ redis: { quit: mocks.quit },+ sessionGetOrCreate: (...args: unknown[]) => mocks.sessionGetOrCreate(...args),+}));++import { SessionsService } from "./sessionsService.js";++const SESSION_TTL_MS = 30 * 60 * 1000;++describe("SessionsService (Redis-backed)", () => {+ let service: SessionsService;++ beforeEach(() => {+ vi.useFakeTimers();+ vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));+ mocks.sessionGetOrCreate.mockReset();+ mocks.quit.mockReset();+ service = new SessionsService();+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("returns the session id resolved by Redis and refreshes its TTL", async () => {+ mocks.sessionGetOrCreate.mockResolvedValue("sess-existing");++ const result = await service.updateSession({ userId: "user-a", siteId: 42 });++ expect(result.sessionId).toBe("sess-existing");+ expect(mocks.sessionGetOrCreate).toHaveBeenCalledTimes(1);+ const [key, candidate, ttl] = mocks.sessionGetOrCreate.mock.calls[0];+ expect(key).toBe("session:42:user-a");+ expect(typeof candidate).toBe("string");+ expect(candidate).toHaveLength(14); // nanoid(14) candidate+ expect(ttl).toBe(SESSION_TTL_MS);+ });++ it("namespaces the Redis key by site and user", async () => {+ mocks.sessionGetOrCreate.mockResolvedValue("x");++ await service.updateSession({ userId: "u1", siteId: 1 });+ await service.updateSession({ userId: "u1", siteId: 2 });+ await service.updateSession({ userId: "u2", siteId: 1 });++ const keys = mocks.sessionGetOrCreate.mock.calls.map(c => c[0]);+ expect(keys).toEqual(["session:1:u1", "session:2:u1", "session:1:u2"]);+ });++ it("falls back to a window-stable id when Redis fails, without throwing", async () => {+ mocks.sessionGetOrCreate.mockRejectedValue(new Error("redis down"));++ const first = await service.updateSession({ userId: "user-b", siteId: 7 });+ // Same 30-minute bucket β same fallback id, so events still share a session.+ vi.setSystemTime(new Date(Date.now() + SESSION_TTL_MS - 1000));+ const second = await service.updateSession({ userId: "user-b", siteId: 7 });++ expect(first.sessionId).toBe(second.sessionId);+ expect(first.sessionId).toContain("7");+ expect(first.sessionId).toContain("user-b");+ });++ it("rotates the fallback id once the time bucket advances", async () => {+ mocks.sessionGetOrCreate.mockRejectedValue(new Error("redis down"));++ const first = await service.updateSession({ userId: "user-c", siteId: 7 });+ vi.setSystemTime(new Date(Date.now() + SESSION_TTL_MS + 1000));+ const later = await service.updateSession({ userId: "user-c", siteId: 7 });++ expect(first.sessionId).not.toBe(later.sessionId);+ });++ it("closes the Redis connection on shutdown", async () => {+ mocks.quit.mockResolvedValue("OK");+ await service.close();+ expect(mocks.quit).toHaveBeenCalledTimes(1);+ });+});diff --git a/server/src/services/sessions/sessionsService.ts b/server/src/services/sessions/sessionsService.tsindex c27ff8a7d..85847026d 100644--- a/server/src/services/sessions/sessionsService.ts+++ b/server/src/services/sessions/sessionsService.ts@@ -1,90 +1,50 @@-import { and, eq, lt } from "drizzle-orm";import { nanoid } from "nanoid";-import * as cron from "node-cron";-import { db } from "../../db/postgres/postgres.js";-import { activeSessions } from "../../db/postgres/schema.js";import { createServiceLogger } from "../../lib/logger/logger.js";+import { redis, sessionGetOrCreate } from "../../db/redis/redis.js";-class SessionsService {- private cleanupTask: cron.ScheduledTask | null = null;- private logger = createServiceLogger("sessions");-- constructor() {}-- private initializeCleanupCron() {- this.cleanupTask = cron.schedule(- "* * * * *",- async () => {- try {- const deletedCount = await this.cleanupOldSessions();- // Uncomment for debugging- this.logger.debug(`Cleaned up ${deletedCount} expired sessions`);- } catch (error) {- this.logger.error(error as Error, "Error during session cleanup");- }- },- { timezone: "UTC" }- );+// Sessions expire after this much inactivity. Redis refreshes the TTL on every+// event (sliding window) and evicts the key automatically once it lapses β there+// is no table to scan and no cleanup cron to run.+const SESSION_TTL_MS = 30 * 60 * 1000;- this.logger.info("Session cleanup cron initialized (runs every minute)");- }- async getExistingSession(userId: string, siteId: number) {- const [existingSession] = await db- .select()- .from(activeSessions)- .where(and(eq(activeSessions.userId, userId), eq(activeSessions.siteId, siteId)))- .limit(1);+export class SessionsService {+ private logger = createServiceLogger("sessions");- return existingSession || null;+ private getSessionKey(userId: string, siteId: number): string {+ return `session:${siteId}:${userId}`;}+ /**+ * Get the active session id for a (userId, siteId) pair, creating one if none+ * exists, and refresh its sliding 30-minute TTL. Backed entirely by Redis.+ */async updateSession({ userId, siteId }: { userId: string; siteId: number }): Promise<{ sessionId: string }> {- const existingSession = await this.getExistingSession(userId, siteId);-- if (existingSession) {- await db- .update(activeSessions)- .set({- lastActivity: new Date(),- })- .where(eq(activeSessions.sessionId, existingSession.sessionId));- return { sessionId: existingSession.sessionId };+ const key = this.getSessionKey(userId, siteId);+ const candidate = nanoid(14);++ try {+ const sessionId = await sessionGetOrCreate(key, candidate, SESSION_TTL_MS);+ return { sessionId };+ } catch (error) {+ // A Redis blip must never drop ingestion. Fall back to a deterministic,+ // window-stable id so events in the same 30-minute bucket still share a+ // session until Redis recovers.+ this.logger.error(error as Error, "Redis session lookup failed; using fallback session id");+ return { sessionId: this.fallbackSessionId(userId, siteId) };}-- const insertData = {- sessionId: nanoid(14),- siteId,- userId,- startTime: new Date(),- lastActivity: new Date(),- };-- await db.insert(activeSessions).values(insertData);- return { sessionId: insertData.sessionId };- }-- async cleanupOldSessions(): Promise<number> {- // Delete sessions older than 30 minutes- const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);-- const deletedSessions = await db- .delete(activeSessions)- .where(lt(activeSessions.lastActivity, thirtyMinutesAgo))- .returning();-- // this.logger.debug(`Cleaned up ${deletedSessions.length} sessions`);- return deletedSessions.length;}- // Method to stop the cleanup cron job (useful for graceful shutdown)- startCleanupCron() {- this.initializeCleanupCron();+ private fallbackSessionId(userId: string, siteId: number): string {+ const bucket = Math.floor(Date.now() / SESSION_TTL_MS);+ return `f_${siteId}_${userId}_${bucket}`;}- stopCleanupCron() {- if (this.cleanupTask) {- this.cleanupTask.stop();- this.logger.info("Session cleanup cron stopped");+ /** Close the Redis connection during graceful shutdown. */+ async close(): Promise<void> {+ try {+ await redis.quit();+ } catch (error) {+ this.logger.error(error as Error, "Error closing Redis connection");}}}diff --git a/server/src/services/tracker/botBlocking/anomalyScorer.test.ts b/server/src/services/tracker/botBlocking/anomalyScorer.test.tsindex a7bc6add8..e3f9431c3 100644--- a/server/src/services/tracker/botBlocking/anomalyScorer.test.ts+++ b/server/src/services/tracker/botBlocking/anomalyScorer.test.ts@@ -1,5 +1,19 @@-import { beforeEach, describe, expect, it } from "vitest";-import { observeTrackingAnomaly, resetAnomalyScorerForTests } from "./anomalyScorer.js";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ anomalyObserve: vi.fn(),+}));++vi.mock("../../../db/redis/redis.js", () => ({+ redis: {},+ anomalyObserve: (...args: unknown[]) => mocks.anomalyObserve(...args),+}));++import {+ observeTrackingAnomaly,+ resetAnomalyScorerForTests,+ setRedisAnomalyEnabledForTests,+} from "./anomalyScorer.js";const baseInput = {siteId: "site_123",@@ -12,36 +26,37 @@ const baseInput = {nowMs: 1_000_000,};-describe("observeTrackingAnomaly", () => {+describe("observeTrackingAnomaly (in-process fallback)", () => {beforeEach(() => {resetAnomalyScorerForTests();+ setRedisAnomalyEnabledForTests(false);});- it("does not flag normal traffic", () => {- const result = observeTrackingAnomaly(baseInput);+ it("does not flag normal traffic", async () => {+ const result = await observeTrackingAnomaly(baseInput);expect(result.isAnomalous).toBe(false);expect(result.score).toBe(0);expect(result.reasons).toEqual([]);});- it("flags high request bursts for a single visitor tuple", () => {- let result = observeTrackingAnomaly(baseInput);+ it("flags high request bursts for a single visitor tuple", async () => {+ let result = await observeTrackingAnomaly(baseInput);for (let i = 1; i < 30; i++) {- result = observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + i });+ result = await observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + i });}expect(result.isAnomalous).toBe(false);- result = observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + 30 });+ result = await observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + 30 });expect(result.isAnomalous).toBe(true);expect(result.reasons.map(reason => reason.rule)).toContain("tuple_events_10s");});- it("flags fast path crawling", () => {- let result = observeTrackingAnomaly(baseInput);+ it("flags fast path crawling", async () => {+ let result = await observeTrackingAnomaly(baseInput);for (let i = 1; i <= 25; i++) {- result = observeTrackingAnomaly({+ result = await observeTrackingAnomaly({...baseInput,pathname: `/docs/${i}`,nowMs: baseInput.nowMs + i,@@ -52,10 +67,10 @@ describe("observeTrackingAnomaly", () => {expect(result.reasons.map(reason => reason.rule)).toContain("tuple_distinct_paths_60s");});- it("treats missing client score as weak context, not enough to block by itself", () => {- let result = observeTrackingAnomaly({ ...baseInput, hasClientBotScore: false });+ it("treats missing client score as weak context, not enough to block by itself", async () => {+ let result = await observeTrackingAnomaly({ ...baseInput, hasClientBotScore: false });for (let i = 1; i <= 20; i++) {- result = observeTrackingAnomaly({+ result = await observeTrackingAnomaly({...baseInput,hasClientBotScore: false,nowMs: baseInput.nowMs + i * 2_000,@@ -74,13 +89,76 @@ describe("observeTrackingAnomaly", () => {]);});- it("expires old observations outside the window", () => {+ it("expires old observations outside the window", async () => {for (let i = 0; i < 35; i++) {- observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + i });+ await observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + i });}- const result = observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + 70_000 });+ const result = await observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + 70_000 });expect(result.isAnomalous).toBe(false);expect(result.counters.tupleEvents10s).toBe(1);});});++describe("observeTrackingAnomaly (Redis-backed)", () => {+ beforeEach(() => {+ resetAnomalyScorerForTests();+ setRedisAnomalyEnabledForTests(true);+ mocks.anomalyObserve.mockReset();+ });++ it("sends one spec per enabled counter and maps results back by counter name", async () => {+ // 8 counters, all enabled (path + host present, no client score).+ mocks.anomalyObserve.mockResolvedValue([31, 5, 2, 9, 3, 1, 4, 7]);++ const result = await observeTrackingAnomaly({ ...baseInput, hasClientBotScore: false });++ expect(mocks.anomalyObserve).toHaveBeenCalledTimes(1);+ const [nowMs, specs] = mocks.anomalyObserve.mock.calls[0];+ expect(nowMs).toBe(baseInput.nowMs);+ expect(specs).toHaveLength(8);+ expect(specs.map((spec: { key: string }) => spec.key)).toEqual([+ expect.stringContaining("bot:a:te10:"),+ expect.stringContaining("bot:a:te60:"),+ expect.stringContaining("bot:a:tdp:"),+ expect.stringContaining("bot:a:ie60:"),+ expect.stringContaining("bot:a:idua:"),+ expect.stringContaining("bot:a:idh:"),+ expect.stringContaining("bot:a:sue:"),+ expect.stringContaining("bot:a:mcs:"),+ ]);++ expect(result.counters.tupleEvents10s).toBe(31);+ expect(result.counters.missingClientScore60s).toBe(7);+ expect(result.isAnomalous).toBe(true);+ expect(result.reasons.map(reason => reason.rule)).toContain("tuple_events_10s");+ });++ it("omits conditional counters that don't apply and reports them as zero", async () => {+ // No pathname, no hostname, client score present β 3 counters dropped.+ mocks.anomalyObserve.mockResolvedValue([1, 1, 1, 1, 1]);++ const result = await observeTrackingAnomaly({+ ...baseInput,+ pathname: undefined,+ hostname: undefined,+ hasClientBotScore: true,+ });++ const [, specs] = mocks.anomalyObserve.mock.calls[0];+ expect(specs).toHaveLength(5);+ expect(result.counters.tupleDistinctPaths60s).toBe(0);+ expect(result.counters.ipDistinctHosts60s).toBe(0);+ expect(result.counters.missingClientScore60s).toBe(0);+ });++ it("falls back to in-process counting when Redis fails, without throwing", async () => {+ mocks.anomalyObserve.mockRejectedValue(new Error("redis down"));++ const first = await observeTrackingAnomaly(baseInput);+ expect(first.counters.tupleEvents10s).toBe(1);++ const second = await observeTrackingAnomaly({ ...baseInput, nowMs: baseInput.nowMs + 1 });+ expect(second.counters.tupleEvents10s).toBe(2);+ });+});diff --git a/server/src/services/tracker/botBlocking/anomalyScorer.ts b/server/src/services/tracker/botBlocking/anomalyScorer.tsindex 7bd4a73b3..e329530ff 100644--- a/server/src/services/tracker/botBlocking/anomalyScorer.ts+++ b/server/src/services/tracker/botBlocking/anomalyScorer.ts@@ -1,10 +1,22 @@-import { createHash } from "crypto";+import { anomalyObserve, type AnomalyCounterSpec } from "../../../db/redis/redis.js";+import { createServiceLogger } from "../../../lib/logger/logger.js";const SECOND = 1000;const MINUTE = 60 * SECOND;const ANOMALY_SCORE_THRESHOLD = 4;const CLEANUP_INTERVAL_MS = 60 * SECOND;+const MAX_COUNTER_BUCKET_SIZE = 512;+const MAX_DISTINCT_BUCKET_SIZE = 512;++const logger = createServiceLogger("anomaly-scorer");++// Counters live in Redis so every worker/replica shares one view; an in-process+// Map only sees 1/N of the traffic behind a load balancer, diluting the rate+// thresholds by the worker count. The in-process counters below remain as a+// fallback for when Redis is unavailable. Set DISABLE_REDIS_ANOMALY=true to pin+// scoring to the in-process counters (e.g. single-process deployments).+let redisAnomalyEnabled = process.env.DISABLE_REDIS_ANOMALY !== "true";interface AnomalyReason {rule: string;@@ -103,6 +115,15 @@ const ipDistinctHosts60s = new RollingDistinctCounter();let lastCleanupMs = 0;+// Unique-per-event token so each observation is a distinct sorted-set member in+// the Redis rate counters. pid keeps it unique across workers; the sequence keeps+// it unique within a millisecond.+let eventSeq = 0;+function nextEventToken(nowMs: number): string {+ eventSeq = (eventSeq + 1) % Number.MAX_SAFE_INTEGER;+ return `${nowMs}-${process.pid}-${eventSeq}`;+}+function pruneTimestamps(bucket: number[], nowMs: number, windowMs: number): number {const oldestAllowed = nowMs - windowMs;let removeCount = 0;@@ -112,6 +133,9 @@ function pruneTimestamps(bucket: number[], nowMs: number, windowMs: number): numif (removeCount > 0) {bucket.splice(0, removeCount);}+ if (bucket.length > MAX_COUNTER_BUCKET_SIZE) {+ bucket.splice(0, bucket.length - MAX_COUNTER_BUCKET_SIZE);+ }return bucket.length;}@@ -122,11 +146,21 @@ function pruneDistinct(bucket: Map<string, number>, nowMs: number, windowMs: numbucket.delete(value);}}+ while (bucket.size > MAX_DISTINCT_BUCKET_SIZE) {+ const oldestKey = bucket.keys().next().value;+ if (oldestKey === undefined) break;+ bucket.delete(oldestKey);+ }return bucket.size;}function hashValue(value: string): string {- return createHash("sha256").update(value).digest("base64url").slice(0, 16);+ let hash = 0x811c9dc5;+ for (let index = 0; index < value.length; index++) {+ hash ^= value.charCodeAt(index);+ hash = Math.imul(hash, 0x01000193);+ }+ return (hash >>> 0).toString(36);}function normalizeDimension(value: string | undefined): string {@@ -159,10 +193,20 @@ function maybeCleanup(nowMs: number) {ipDistinctHosts60s.cleanup(nowMs, MINUTE);}-export function observeTrackingAnomaly(input: AnomalyInput): AnomalyResult {- const nowMs = input.nowMs ?? Date.now();- maybeCleanup(nowMs);+// A single counter described once for both backends: its Redis key/member and an+// equivalent in-process observation. `enabled` is false for the conditional+// counters (no path/host, or a client score was supplied), which contribute 0.+interface CounterPlan {+ name: keyof AnomalyCounters;+ enabled: boolean;+ redisKey: string;+ member: string;+ windowMs: number;+ maxSize: number;+ observeLocal: (nowMs: number) => number;+}+function buildCounterPlan(input: AnomalyInput, nowMs: number): CounterPlan[] {const siteId = input.siteId;const ipAddress = normalizeDimension(input.ipAddress);const userAgentHash = hashValue(input.userAgent || "");@@ -172,18 +216,127 @@ export function observeTrackingAnomaly(input: AnomalyInput): AnomalyResult {const tupleKey = `${siteId}:${ipAddress}:${userAgentHash}`;const ipKey = `${siteId}:${ipAddress}`;const siteUserAgentKey = `${siteId}:${userAgentHash}`;+ const eventToken = nextEventToken(nowMs);++ return [+ {+ name: "tupleEvents10s",+ enabled: true,+ redisKey: `bot:a:te10:${tupleKey}`,+ member: eventToken,+ windowMs: 10 * SECOND,+ maxSize: MAX_COUNTER_BUCKET_SIZE,+ observeLocal: now => tupleEvents10s.observe(tupleKey, now, 10 * SECOND),+ },+ {+ name: "tupleEvents60s",+ enabled: true,+ redisKey: `bot:a:te60:${tupleKey}`,+ member: eventToken,+ windowMs: MINUTE,+ maxSize: MAX_COUNTER_BUCKET_SIZE,+ observeLocal: now => tupleEvents60s.observe(tupleKey, now, MINUTE),+ },+ {+ name: "tupleDistinctPaths60s",+ enabled: pathname !== "",+ redisKey: `bot:a:tdp:${tupleKey}`,+ member: pathname,+ windowMs: MINUTE,+ maxSize: MAX_DISTINCT_BUCKET_SIZE,+ observeLocal: now => tupleDistinctPaths60s.observe(tupleKey, pathname, now, MINUTE),+ },+ {+ name: "ipEvents60s",+ enabled: true,+ redisKey: `bot:a:ie60:${ipKey}`,+ member: eventToken,+ windowMs: MINUTE,+ maxSize: MAX_COUNTER_BUCKET_SIZE,+ observeLocal: now => ipEvents60s.observe(ipKey, now, MINUTE),+ },+ {+ name: "ipDistinctUserAgents5m",+ enabled: true,+ redisKey: `bot:a:idua:${ipKey}`,+ member: userAgentHash,+ windowMs: 5 * MINUTE,+ maxSize: MAX_DISTINCT_BUCKET_SIZE,+ observeLocal: now => ipDistinctUserAgents5m.observe(ipKey, userAgentHash, now, 5 * MINUTE),+ },+ {+ name: "ipDistinctHosts60s",+ enabled: hostname !== "",+ redisKey: `bot:a:idh:${ipKey}`,+ member: hostname,+ windowMs: MINUTE,+ maxSize: MAX_DISTINCT_BUCKET_SIZE,+ observeLocal: now => ipDistinctHosts60s.observe(ipKey, hostname, now, MINUTE),+ },+ {+ name: "siteUserAgentEvents60s",+ enabled: true,+ redisKey: `bot:a:sue:${siteUserAgentKey}`,+ member: eventToken,+ windowMs: MINUTE,+ maxSize: MAX_COUNTER_BUCKET_SIZE,+ observeLocal: now => siteUserAgentEvents60s.observe(siteUserAgentKey, now, MINUTE),+ },+ {+ name: "missingClientScore60s",+ enabled: !input.hasClientBotScore,+ redisKey: `bot:a:mcs:${tupleKey}`,+ member: eventToken,+ windowMs: MINUTE,+ maxSize: MAX_COUNTER_BUCKET_SIZE,+ observeLocal: now => missingClientScore60s.observe(tupleKey, now, MINUTE),+ },+ ];+}- const counters: AnomalyCounters = {- tupleEvents10s: tupleEvents10s.observe(tupleKey, nowMs, 10 * SECOND),- tupleEvents60s: tupleEvents60s.observe(tupleKey, nowMs, MINUTE),- tupleDistinctPaths60s: pathname ? tupleDistinctPaths60s.observe(tupleKey, pathname, nowMs, MINUTE) : 0,- ipEvents60s: ipEvents60s.observe(ipKey, nowMs, MINUTE),- ipDistinctUserAgents5m: ipDistinctUserAgents5m.observe(ipKey, userAgentHash, nowMs, 5 * MINUTE),- ipDistinctHosts60s: hostname ? ipDistinctHosts60s.observe(ipKey, hostname, nowMs, MINUTE) : 0,- siteUserAgentEvents60s: siteUserAgentEvents60s.observe(siteUserAgentKey, nowMs, MINUTE),- missingClientScore60s: input.hasClientBotScore ? 0 : missingClientScore60s.observe(tupleKey, nowMs, MINUTE),+function emptyCounters(): AnomalyCounters {+ return {+ tupleEvents10s: 0,+ tupleEvents60s: 0,+ tupleDistinctPaths60s: 0,+ ipEvents60s: 0,+ ipDistinctUserAgents5m: 0,+ ipDistinctHosts60s: 0,+ siteUserAgentEvents60s: 0,+ missingClientScore60s: 0,};+}++async function observeViaRedis(plan: CounterPlan[], nowMs: number): Promise<AnomalyCounters> {+ const enabled = plan.filter(entry => entry.enabled);+ const specs: AnomalyCounterSpec[] = enabled.map(entry => ({+ key: entry.redisKey,+ member: entry.member,+ windowMs: entry.windowMs,+ maxSize: entry.maxSize,+ }));++ const results = await anomalyObserve(nowMs, specs);++ const counters = emptyCounters();+ enabled.forEach((entry, index) => {+ counters[entry.name] = results[index] ?? 0;+ });+ return counters;+}+function observeViaLocal(plan: CounterPlan[], nowMs: number): AnomalyCounters {+ maybeCleanup(nowMs);+ const counters = emptyCounters();+ for (const entry of plan) {+ if (entry.enabled) {+ counters[entry.name] = entry.observeLocal(nowMs);+ }+ }+ return counters;+}++function computeAnomalyResult(counters: AnomalyCounters): AnomalyResult {const reasons: AnomalyReason[] = [];addReason(reasons, "tuple_events_10s", 4, counters.tupleEvents10s, 30, 10);addReason(reasons, "tuple_events_60s", 4, counters.tupleEvents60s, 120, 60);@@ -204,6 +357,28 @@ export function observeTrackingAnomaly(input: AnomalyInput): AnomalyResult {};}+export async function observeTrackingAnomaly(input: AnomalyInput): Promise<AnomalyResult> {+ const nowMs = input.nowMs ?? Date.now();+ const plan = buildCounterPlan(input, nowMs);++ let counters: AnomalyCounters;+ if (redisAnomalyEnabled) {+ try {+ counters = await observeViaRedis(plan, nowMs);+ } catch (error) {+ // A Redis blip must never break ingestion. Fall back to the in-process+ // counters β accuracy degrades under clustering, but detection keeps working+ // and recovers automatically once Redis is back.+ logger.error(error as Error, "Redis anomaly counters failed; using in-process fallback");+ counters = observeViaLocal(plan, nowMs);+ }+ } else {+ counters = observeViaLocal(plan, nowMs);+ }++ return computeAnomalyResult(counters);+}+export function resetAnomalyScorerForTests() {tupleEvents10s.clear();tupleEvents60s.clear();@@ -214,4 +389,10 @@ export function resetAnomalyScorerForTests() {ipDistinctUserAgents5m.clear();ipDistinctHosts60s.clear();lastCleanupMs = 0;+ eventSeq = 0;+}++/** Force the counter backend in tests (Redis vs. in-process fallback). */+export function setRedisAnomalyEnabledForTests(enabled: boolean) {+ redisAnomalyEnabled = enabled;}diff --git a/server/src/services/tracker/botBlocking/botDetectionStats.test.ts b/server/src/services/tracker/botBlocking/botDetectionStats.test.tsnew file mode 100644index 000000000..564c18011--- /dev/null+++ b/server/src/services/tracker/botBlocking/botDetectionStats.test.ts@@ -0,0 +1,89 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ hincrby: vi.fn(),+ exec: vi.fn(),+ set: vi.fn(),+ hgetall: vi.fn(),+}));++vi.mock("../../../db/redis/redis.js", () => ({+ redis: {+ pipeline: () => ({ hincrby: mocks.hincrby, exec: mocks.exec }),+ set: (...args: unknown[]) => mocks.set(...args),+ hgetall: (...args: unknown[]) => mocks.hgetall(...args),+ },+}));++import {+ flushBotDetectionStats,+ recordBotBlockingRequest,+ recordBotDetections,+ resetBotDetectionStatsForTests,+} from "./botDetectionStats.js";++function hincrbyField(field: string): number | undefined {+ const call = mocks.hincrby.mock.calls.find(c => c[1] === field);+ return call?.[2] as number | undefined;+}++describe("botDetectionStats aggregation", () => {+ beforeEach(() => {+ resetBotDetectionStatsForTests();+ mocks.hincrby.mockReset();+ mocks.exec.mockReset().mockResolvedValue([]);+ mocks.set.mockReset().mockResolvedValue(null);+ mocks.hgetall.mockReset().mockResolvedValue({});+ });++ it("flushes only the delta since the previous flush", async () => {+ recordBotBlockingRequest(undefined, undefined); // totalRequests=1, cs:missing=1, sig:missingMask=1+ recordBotDetections(["ua_pattern"]); // totalBotRequests=1, m:ua_pattern=1++ await flushBotDetectionStats();++ expect(hincrbyField("totalRequests")).toBe(1);+ expect(hincrbyField("totalBotRequests")).toBe(1);+ expect(hincrbyField("m:ua_pattern")).toBe(1);+ expect(hincrbyField("cs:missing")).toBe(1);+ expect(mocks.exec).toHaveBeenCalledTimes(1);++ // Second flush with one more request should only push the new delta.+ mocks.hincrby.mockReset();+ recordBotBlockingRequest(0, 0);+ await flushBotDetectionStats();++ expect(hincrbyField("totalRequests")).toBe(1); // delta, not the running total of 2+ expect(hincrbyField("cs:score0")).toBe(1);+ expect(hincrbyField("m:ua_pattern")).toBeUndefined(); // unchanged β not sent+ });++ it("does not call exec when there is nothing new to flush", async () => {+ await flushBotDetectionStats();+ expect(mocks.hincrby).not.toHaveBeenCalled();+ expect(mocks.exec).not.toHaveBeenCalled();+ });++ it("logs the aggregate only when it wins the per-interval lock", async () => {+ recordBotBlockingRequest(undefined, undefined);++ mocks.set.mockResolvedValue(null); // lock not acquired+ await flushBotDetectionStats();+ expect(mocks.hgetall).not.toHaveBeenCalled();++ mocks.set.mockResolvedValue("OK"); // lock acquired+ recordBotBlockingRequest(0, 0);+ await flushBotDetectionStats();+ expect(mocks.hgetall).toHaveBeenCalledWith("bot:stats");++ // Lock attempt uses NX so only one worker wins per interval.+ expect(mocks.set).toHaveBeenLastCalledWith("bot:stats:loglock", "1", "PX", expect.any(Number), "NX");+ });++ it("does not throw when Redis is unavailable", async () => {+ recordBotDetections(["rate_anomaly"]);+ mocks.exec.mockRejectedValue(new Error("redis down"));++ await expect(flushBotDetectionStats()).resolves.toBeUndefined();+ });+});diff --git a/server/src/services/tracker/botBlocking/botDetectionStats.ts b/server/src/services/tracker/botBlocking/botDetectionStats.tsindex 223e97b42..27ac02127 100644--- a/server/src/services/tracker/botBlocking/botDetectionStats.ts+++ b/server/src/services/tracker/botBlocking/botDetectionStats.ts@@ -1,3 +1,4 @@+import { redis } from "../../../db/redis/redis.js";import { logger } from "../../../lib/logger/logger.js";export type BotDetectionMethod = "ua_pattern" | "header_heuristics" | "client_signals" | "bot_asn" | "rate_anomaly";@@ -10,6 +11,17 @@ const BOT_DETECTION_METHODS: readonly BotDetectionMethod[] = ["rate_anomaly",];+const BOT_DETECTION_STATS_INTERVAL_MS = 60_000;++// Counters are incremented in-process on the hot path (no per-event Redis), then+// flushed as deltas into a shared hash once per interval so the logged totals are+// cluster-wide rather than one fragmented slice per worker.+const STATS_HASH_KEY = "bot:stats";+// Single-logger election: whichever worker wins this short-lived lock logs the+// aggregate for the interval, so the cluster emits one line instead of N.+const STATS_LOG_LOCK_KEY = "bot:stats:loglock";+const STATS_LOG_LOCK_TTL_MS = BOT_DETECTION_STATS_INTERVAL_MS - 5_000;+const totals: Record<BotDetectionMethod, number> = {ua_pattern: 0,header_heuristics: 0,@@ -59,14 +71,19 @@ const clientBotSignalTotals: Record<ClientBotSignalTotal, number> = {unknownMaskBits: 0,};+// Canonical field sets, captured once, used to round-trip counters through the+// flat Redis hash.+const HISTOGRAM_KEYS = Object.keys(clientBotScoreHistogram) as (keyof typeof clientBotScoreHistogram)[];+const SIGNAL_KEYS = Object.keys(clientBotSignalTotals) as ClientBotSignalTotal[];+let totalRequests = 0;let totalBotRequests = 0;-function getBotRequestPercentage() {- if (totalRequests === 0) {+function getBotRequestPercentage(requests = totalRequests, botRequests = totalBotRequests) {+ if (requests === 0) {return 0;}- return Number(((totalBotRequests / totalRequests) * 100).toFixed(2));+ return Number(((botRequests / requests) * 100).toFixed(2));}export function recordBotBlockingRequest(clientBotScore: number | undefined, clientBotSignalMask: number | undefined) {@@ -123,37 +140,116 @@ export function resetBotDetectionStatsForTests() {for (const method of BOT_DETECTION_METHODS) {totals[method] = 0;}- clientBotScoreHistogram.missing = 0;- clientBotScoreHistogram.score0 = 0;- clientBotScoreHistogram.score1 = 0;- clientBotScoreHistogram.score2 = 0;- clientBotScoreHistogram.score3Plus = 0;- clientBotSignalTotals.missingMask = 0;- clientBotSignalTotals.automationApi = 0;- clientBotSignalTotals.zeroOuterDimensions = 0;- clientBotSignalTotals.missingChrome = 0;- clientBotSignalTotals.swiftShader = 0;- clientBotSignalTotals.emptyPlugins = 0;- clientBotSignalTotals.defaultViewport800x600 = 0;- clientBotSignalTotals.defaultViewport1024x768 = 0;- clientBotSignalTotals.impossibleDimensions = 0;- clientBotSignalTotals.outerDimensionsWeird = 0;- clientBotSignalTotals.pluginApiAbsence = 0;- clientBotSignalTotals.unknownMaskBits = 0;+ for (const key of HISTOGRAM_KEYS) {+ clientBotScoreHistogram[key] = 0;+ }+ for (const key of SIGNAL_KEYS) {+ clientBotSignalTotals[key] = 0;+ }+ lastFlushed = {};+}++// Flatten the in-process counters to the hash field names used in Redis.+function flattenLocalCounters(): Record<string, number> {+ const flat: Record<string, number> = {+ totalRequests,+ totalBotRequests,+ };+ for (const method of BOT_DETECTION_METHODS) {+ flat[`m:${method}`] = totals[method];+ }+ for (const key of HISTOGRAM_KEYS) {+ flat[`cs:${key}`] = clientBotScoreHistogram[key];+ }+ for (const key of SIGNAL_KEYS) {+ flat[`sig:${key}`] = clientBotSignalTotals[key];+ }+ return flat;+}++// Reassemble a hash read from Redis into the structured shape we log.+function structureAggregate(hash: Record<string, string>) {+ const num = (field: string) => Number(hash[field] ?? 0) || 0;++ const aggregateTotals = {} as Record<BotDetectionMethod, number>;+ for (const method of BOT_DETECTION_METHODS) {+ aggregateTotals[method] = num(`m:${method}`);+ }++ const histogram = {} as Record<keyof typeof clientBotScoreHistogram, number>;+ for (const key of HISTOGRAM_KEYS) {+ histogram[key] = num(`cs:${key}`);+ }++ const signalTotals = {} as Record<ClientBotSignalTotal, number>;+ for (const key of SIGNAL_KEYS) {+ signalTotals[key] = num(`sig:${key}`);+ }++ const requests = num("totalRequests");+ const botRequests = num("totalBotRequests");++ return {+ totalRequests: requests,+ totalBotRequests: botRequests,+ botRequestPercentage: getBotRequestPercentage(requests, botRequests),+ botDetectionTotals: aggregateTotals,+ clientBotScoreHistogram: histogram,+ clientBotSignalTotals: signalTotals,+ };+}++// Tracks what we've already pushed to Redis so each flush only sends the delta+// since the last flush.+let lastFlushed: Record<string, number> = {};+let flushing = false;++/**+ * Flush the in-process counter deltas into the shared Redis hash and, if this+ * process wins the per-interval lock, log the cluster-wide aggregate. Safe to+ * call concurrently (re-entrant calls are ignored) and never throws.+ */+export async function flushBotDetectionStats(): Promise<void> {+ if (flushing) return;+ flushing = true;+ try {+ const current = flattenLocalCounters();++ const pipeline = redis.pipeline();+ let hasDelta = false;+ for (const [field, value] of Object.entries(current)) {+ const delta = value - (lastFlushed[field] ?? 0);+ if (delta !== 0) {+ pipeline.hincrby(STATS_HASH_KEY, field, delta);+ hasDelta = true;+ }+ }+ if (hasDelta) {+ await pipeline.exec();+ }+ lastFlushed = current;++ // Only the worker that wins the lock logs, so the cluster emits one+ // aggregated line per interval instead of one per worker.+ const acquired = await redis.set(STATS_LOG_LOCK_KEY, "1", "PX", STATS_LOG_LOCK_TTL_MS, "NX");+ if (acquired) {+ const hash = await redis.hgetall(STATS_HASH_KEY);+ logger.info(structureAggregate(hash), "Bot detection totals (cluster, cumulative)");+ }+ } catch (error) {+ // Redis aggregation unavailable β don't go dark. Log this process's local+ // totals so observability degrades gracefully rather than disappearing.+ logger.warn(+ { err: error as Error, ...getBotDetectionStats() },+ "Bot detection totals (local; Redis aggregation unavailable)"+ );+ } finally {+ flushing = false;+ }}const interval = setInterval(() => {- logger.info(- {- totalRequests,- totalBotRequests,- botRequestPercentage: getBotRequestPercentage(),- botDetectionTotals: { ...totals },- clientBotScoreHistogram: { ...clientBotScoreHistogram },- clientBotSignalTotals: { ...clientBotSignalTotals },- },- "Bot detection totals since server start"- );-}, 5_000);+ void flushBotDetectionStats();+}, BOT_DETECTION_STATS_INTERVAL_MS);interval.unref?.();diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.tsindex f32d8abdc..207ce1618 100644--- a/server/src/services/tracker/botBlocking/botEventQueue.ts+++ b/server/src/services/tracker/botBlocking/botEventQueue.ts@@ -11,10 +11,13 @@ type BotEventPayload = TotalTrackingPayload &sessionId: string;};+const BOT_EVENT_BATCH_SIZE = 5000;+const BOT_EVENT_FLUSH_INTERVAL_MS = 1000;+class BotEventQueue {private queue: BotEventPayload[] = [];- private batchSize = 5000;- private interval = 1000;+ private batchSize = BOT_EVENT_BATCH_SIZE;+ private interval = BOT_EVENT_FLUSH_INTERVAL_MS;private processing = false;private logger = createServiceLogger("bot-event-queue");diff --git a/server/src/services/tracker/botBlocking/index.test.ts b/server/src/services/tracker/botBlocking/index.test.tsindex 02f21c695..eecb1a87b 100644--- a/server/src/services/tracker/botBlocking/index.test.ts+++ b/server/src/services/tracker/botBlocking/index.test.ts@@ -1,7 +1,7 @@import { FastifyRequest } from "fastify";import { beforeEach, describe, expect, it, vi } from "vitest";import { lookupAsn } from "../../../db/geolocation/asn.js";-import { resetAnomalyScorerForTests } from "./anomalyScorer.js";+import { resetAnomalyScorerForTests, setRedisAnomalyEnabledForTests } from "./anomalyScorer.js";import { getBotDetectionStats, resetBotDetectionStatsForTests } from "./botDetectionStats.js";import { checkBotBlocking } from "./index.js";@@ -9,6 +9,13 @@ vi.mock("../../../db/geolocation/asn.js", () => ({lookupAsn: vi.fn(() => null),}));+// Drive anomaly scoring through the in-process counters so detection is+// deterministic and these tests never touch a real Redis.+vi.mock("../../../db/redis/redis.js", () => ({+ redis: {},+ anomalyObserve: vi.fn(),+}));+function requestWithHeaders(headers: Record<string, string | string[]>): FastifyRequest {return { headers } as unknown as FastifyRequest;}@@ -35,12 +42,13 @@ const clientBotSignalMasks = {describe("checkBotBlocking", () => {beforeEach(() => {resetAnomalyScorerForTests();+ setRedisAnomalyEnabledForTests(false);resetBotDetectionStatsForTests();vi.mocked(lookupAsn).mockReturnValue(null);});- it("does nothing when bot blocking is disabled", () => {- const result = checkBotBlocking({+ it("does nothing when bot blocking is disabled", async () => {+ const result = await checkBotBlocking({request: requestWithHeaders({}),blockBots: false,payload: basePayload,@@ -49,8 +57,8 @@ describe("checkBotBlocking", () => {expect(result).toBeNull();});- it("skips verified trusted server-side ingestion requests", () => {- const result = checkBotBlocking({+ it("skips verified trusted server-side ingestion requests", async () => {+ const result = await checkBotBlocking({request: requestWithHeaders({}),blockBots: true,trustedServerSideIngestion: true,@@ -60,8 +68,8 @@ describe("checkBotBlocking", () => {expect(result).toBeNull();});- it("does not bypass bot blocking for an unverified bearer header", () => {- const result = checkBotBlocking({+ it("does not bypass bot blocking for an unverified bearer header", async () => {+ const result = await checkBotBlocking({request: requestWithHeaders({ authorization: "Bearer token" }),blockBots: true,payload: basePayload,@@ -73,13 +81,13 @@ describe("checkBotBlocking", () => {});});- it("counts requests before bot-blocking bypasses for percentage stats", () => {- checkBotBlocking({+ it("counts requests before bot-blocking bypasses for percentage stats", async () => {+ await checkBotBlocking({request: requestWithHeaders({}),blockBots: false,payload: basePayload,});- checkBotBlocking({+ await checkBotBlocking({request: requestWithHeaders({}),blockBots: true,trustedServerSideIngestion: true,@@ -100,8 +108,8 @@ describe("checkBotBlocking", () => {});});- it("returns bot event properties for detected bots", () => {- const result = checkBotBlocking({+ it("returns bot event properties for detected bots", async () => {+ const result = await checkBotBlocking({request: requestWithHeaders({}),blockBots: true,payload: basePayload,@@ -133,25 +141,25 @@ describe("checkBotBlocking", () => {});});- it("records client bot score and signal aggregates for inspected requests", () => {+ it("records client bot score and signal aggregates for inspected requests", async () => {const request = requestWithHeaders(browserHeaders);- checkBotBlocking({+ await checkBotBlocking({request,blockBots: true,payload: basePayload,});- checkBotBlocking({+ await checkBotBlocking({request,blockBots: true,payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },});- checkBotBlocking({+ await checkBotBlocking({request,blockBots: true,payload: { ...basePayload, clientBotScore: 1, clientBotSignalMask: clientBotSignalMasks.swiftShader },});- checkBotBlocking({+ await checkBotBlocking({request,blockBots: true,payload: {@@ -160,7 +168,7 @@ describe("checkBotBlocking", () => {clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions,},});- checkBotBlocking({+ await checkBotBlocking({request,blockBots: true,payload: { ...basePayload, clientBotScore: 3, clientBotSignalMask: clientBotSignalMasks.automationApi },@@ -194,8 +202,8 @@ describe("checkBotBlocking", () => {});});- it("collects every matching bot signal before returning", () => {- const result = checkBotBlocking({+ it("collects every matching bot signal before returning", async () => {+ const result = await checkBotBlocking({request: requestWithHeaders({"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36",@@ -228,8 +236,8 @@ describe("checkBotBlocking", () => {]);});- it("moves default viewport fingerprints into client signals", () => {- const result = checkBotBlocking({+ it("moves default viewport fingerprints into client signals", async () => {+ const result = await checkBotBlocking({request: requestWithHeaders({...browserHeaders,"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120 Safari/537.36",@@ -257,13 +265,13 @@ describe("checkBotBlocking", () => {});});- it("does not block generic hosting ASN by itself", () => {+ it("does not block generic hosting ASN by itself", async () => {vi.mocked(lookupAsn).mockReturnValue({asn: 16509,organization: "Amazon.com, Inc.",});- const result = checkBotBlocking({+ const result = await checkBotBlocking({request: requestWithHeaders(browserHeaders),blockBots: true,payload: {@@ -284,13 +292,13 @@ describe("checkBotBlocking", () => {});});- it("uses generic hosting ASN as supporting evidence when another layer matched", () => {+ it("uses generic hosting ASN as supporting evidence when another layer matched", async () => {vi.mocked(lookupAsn).mockReturnValue({asn: 16509,organization: "Amazon.com, Inc.",});- const result = checkBotBlocking({+ const result = await checkBotBlocking({request: requestWithHeaders(browserHeaders),blockBots: true,payload: {@@ -313,13 +321,13 @@ describe("checkBotBlocking", () => {expect(result?.detections.map(detection => detection.layer)).toEqual(["client_signals", "bot_asn"]);});- it("still blocks curated bot provider ASNs without another matched layer", () => {+ it("still blocks curated bot provider ASNs without another matched layer", async () => {vi.mocked(lookupAsn).mockReturnValue({asn: 401518,organization: "OpenAI, L.L.C.",});- const result = checkBotBlocking({+ const result = await checkBotBlocking({request: requestWithHeaders(browserHeaders),blockBots: true,payload: {@@ -341,12 +349,12 @@ describe("checkBotBlocking", () => {expect(result?.detections.map(detection => detection.layer)).toEqual(["bot_asn"]);});- it("adds a rate anomaly layer after a request burst", () => {+ it("adds a rate anomaly layer after a request burst", async () => {const request = requestWithHeaders(browserHeaders);- let result: ReturnType<typeof checkBotBlocking> = null;+ let result: Awaited<ReturnType<typeof checkBotBlocking>> = null;for (let i = 0; i < 31; i++) {- result = checkBotBlocking({+ result = await checkBotBlocking({request,blockBots: true,payload: {diff --git a/server/src/services/tracker/botBlocking/index.ts b/server/src/services/tracker/botBlocking/index.tsindex 986b42652..7ff6789c8 100644--- a/server/src/services/tracker/botBlocking/index.ts+++ b/server/src/services/tracker/botBlocking/index.ts@@ -10,6 +10,9 @@ import { CLIENT_BOT_SCORE_THRESHOLD } from "./config.js";import { detectBot } from "./headerHeuristics.js";import { classifyUA } from "./uaBots/index.js";+// Per-detection logging is verbose and costly at high traffic; off by default.+const LOG_BOT_DETECTIONS = false;+interface BotBlockingPayload {siteId: string;userAgent?: string;@@ -174,12 +177,12 @@ function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {};}-export function checkBotBlocking({+export async function checkBotBlocking({request,blockBots,trustedServerSideIngestion = false,payload,-}: BotBlockingInput): BotDetectionResult | null {+}: BotBlockingInput): Promise<BotDetectionResult | null> {const userAgent = payload.userAgent || (request.headers["user-agent"] as string) || "";const clientSignalResult = getClientSignalResult(payload, userAgent);recordBotBlockingRequest(clientSignalResult.scoreForStats, clientSignalResult.maskForStats);@@ -253,7 +256,7 @@ export function checkBotBlocking({}// Layer 5: Request-rate and crawl-shape anomaly detection.- const anomaly = observeTrackingAnomaly({+ const anomaly = await observeTrackingAnomaly({siteId: payload.siteId,ipAddress: payload.ipAddress,userAgent,@@ -279,15 +282,17 @@ export function checkBotBlocking({return null;}- logger.info(- {- siteId: payload.siteId,- detectionCount: detections.length,- detectionLayers: detections.map(detection => detection.layer),- detections,- },- "Bot request detected"- );+ if (LOG_BOT_DETECTIONS) {+ logger.info(+ {+ siteId: payload.siteId,+ detectionCount: detections.length,+ detectionLayers: detections.map(detection => detection.layer),+ detections,+ },+ "Bot request detected"+ );+ }recordBotDetections(detections.map(detection => detection.layer));diff --git a/server/src/services/tracker/botBlocking/uaBots/index.ts b/server/src/services/tracker/botBlocking/uaBots/index.tsindex 1c88b0ddc..28b3fab82 100644--- a/server/src/services/tracker/botBlocking/uaBots/index.ts+++ b/server/src/services/tracker/botBlocking/uaBots/index.ts@@ -10,6 +10,13 @@ export interface BotClassification {const NON_BOT: BotClassification = { isBot: false, category: null, matchedPattern: null };+// classifyUA runs on every event (when bot blocking is enabled) and, for bot+// user-agents, scans every compiled pattern. User-agent strings repeat heavily,+// so memoize the classification in a bounded LRU-ish cache to keep the regex+// scan off the per-event hot path. Mirrors parseUserAgent in tracker/utils.ts.+const CLASSIFY_CACHE_MAX = 10_000;+const classifyCache = new Map<string, BotClassification>();+// Single combined regex for the fast "is this anything at all?" test.// Mirrors what isbot does internally.const COMBINED_REGEX = new RegExp(ALL_BOT_PATTERNS.map(p => p.pattern).join("|"), "i");@@ -34,6 +41,25 @@ export function classifyUA(userAgent: string | null | undefined): BotClassificatif (typeof userAgent !== "string" || userAgent.length === 0) {return NON_BOT;}++ const cached = classifyCache.get(userAgent);+ if (cached) {+ // Refresh recency so frequently-seen UAs survive eviction.+ classifyCache.delete(userAgent);+ classifyCache.set(userAgent, cached);+ return cached;+ }++ const result = computeClassification(userAgent);+ classifyCache.set(userAgent, result);+ if (classifyCache.size > CLASSIFY_CACHE_MAX) {+ const oldest = classifyCache.keys().next().value;+ if (oldest !== undefined) classifyCache.delete(oldest);+ }+ return result;+}++function computeClassification(userAgent: string): BotClassification {// Fast path: combined regex test. ~95% of legitimate traffic exits here.if (!COMBINED_REGEX.test(userAgent)) {return NON_BOT;diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.tsindex 5444f4b5b..bff48641b 100644--- a/server/src/services/tracker/pageviewQueue.ts+++ b/server/src/services/tracker/pageviewQueue.ts@@ -10,6 +10,9 @@ type TotalPayload = TotalTrackingPayload & {sessionId: string;};+const PAGEVIEW_BATCH_SIZE = 5000;+const PAGEVIEW_FLUSH_INTERVAL_MS = 1000;+const getParsedProperties = (properties: string | undefined) => {try {return properties ? JSON.parse(properties) : undefined;@@ -20,8 +23,8 @@ const getParsedProperties = (properties: string | undefined) => {class PageviewQueue {private queue: TotalPayload[] = [];- private batchSize = 5000;- private interval = 1000;+ private batchSize = PAGEVIEW_BATCH_SIZE;+ private interval = PAGEVIEW_FLUSH_INTERVAL_MS;private processing = false;private logger = createServiceLogger("pageview-queue");diff --git a/server/src/services/tracker/trackEvent.ts b/server/src/services/tracker/trackEvent.tsindex 11c98d7c1..b845a798a 100644--- a/server/src/services/tracker/trackEvent.ts+++ b/server/src/services/tracker/trackEvent.ts@@ -289,7 +289,7 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {const trackingIdentity = resolveTrackingIdentity(request, validatedPayload, trustedServerSideIngestion);const requestIP = trackingIdentity.ipAddress;- const botDetectionResult = checkBotBlocking({+ const botDetectionResult = await checkBotBlocking({request,blockBots: siteConfiguration.blockBots,trustedServerSideIngestion,@@ -354,17 +354,11 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {trustedServerSideIngestion);- // Update session (use numeric siteId)- const { sessionId } = await sessionsService.updateSession({- userId: payload.userId,- siteId: siteConfiguration.siteId,- });-if (botDetectionResult) {await botEventQueue.add({...payload,...botDetectionResult.eventProperties,- sessionId,+ sessionId: `bot:${payload.userId}`,});return reply.status(200).send({@@ -372,6 +366,12 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {});}+ // Update session (use numeric siteId)+ const { sessionId } = await sessionsService.updateSession({+ userId: payload.userId,+ siteId: siteConfiguration.siteId,+ });+// Add to queue for processing (payload already has numeric siteId)await pageviewQueue.add({...payload,diff --git a/server/src/services/tracker/utils.ts b/server/src/services/tracker/utils.tsindex 5a681f714..4f69aa116 100644--- a/server/src/services/tracker/utils.ts+++ b/server/src/services/tracker/utils.ts@@ -31,6 +31,31 @@ export type TotalTrackingPayload = TrackingPayload & {// Infer type from Zod schematype ValidatedTrackingPayload = z.infer<typeof trackingPayloadSchema>;+// ua-parser-js is regex-heavy and runs on every event. User-agent strings repeat+// constantly across requests, so memoize parsing in a bounded LRU-ish cache to+// keep this off the per-event hot path.+const UA_CACHE_MAX = 10_000;+const uaCache = new Map<string, UAParser.IResult>();++export function parseUserAgent(userAgent: string): UAParser.IResult {+ const cached = uaCache.get(userAgent);+ if (cached) {+ // Refresh recency: re-insert so frequently-seen UAs survive eviction.+ uaCache.delete(userAgent);+ uaCache.set(userAgent, cached);+ return cached;+ }++ const result = userAgentParser(userAgent);+ uaCache.set(userAgent, result);+ if (uaCache.size > UA_CACHE_MAX) {+ // Evict the oldest (least recently used) entry.+ const oldest = uaCache.keys().next().value;+ if (oldest !== undefined) uaCache.delete(oldest);+ }+ return result;+}+// UTM and URL parameter parsing utilitiesexport function getUTMParams(querystring: string): Record<string, string> {const params: Record<string, string> = {};@@ -134,7 +159,7 @@ export async function createBasePayload(type: eventType,ipAddress: ipAddress,timestamp: new Date().toISOString(),- ua: userAgentParser(userAgent),+ ua: parseUserAgent(userAgent),userAgent,userId: anonymousId, // Always the device fingerprintidentifiedUserId: identifiedUserId, // Custom user ID when identified