Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor Redis session management to improve resilience and performance

Bill Yange61127e5cf41
diff --git a/server/src/db/redis/redis.ts b/server/src/db/redis/redis.ts
index 0258acef0..ead957729 100644
--- a/server/src/db/redis/redis.ts
+++ b/server/src/db/redis/redis.ts
@@ -6,26 +6,43 @@ const logger = createServiceLogger("redis");
// Fail fast so callers can fall back rather than hanging during an outage.
const REDIS_COMMAND_TIMEOUT_MS = 1000;
-// A request/response Redis client shared across the process. Distinct from the
-// BullMQ connections used by the uptime service (those have their own settings).
-export const redis = new Redis({
- host: process.env.REDIS_HOST || "localhost",
- port: parseInt(process.env.REDIS_PORT || "6379", 10),
- ...(process.env.REDIS_PASSWORD ? { password: process.env.REDIS_PASSWORD } : {}),
- commandTimeout: REDIS_COMMAND_TIMEOUT_MS,
- maxRetriesPerRequest: 3,
- retryStrategy: (times: number) => Math.min(times * 200, 5000),
-});
+// ioredis pipelines commands over a single connection, so a slow command holds
+// up everything queued behind it (head-of-line blocking). Build each hot-path
+// concern its own connection so, for example, session resolution never waits on
+// the heavier multi-key anomaly Lua script that runs on the same event.
+function createRedisClient(label: string): Redis {
+ const client = new Redis({
+ host: process.env.REDIS_HOST || "localhost",
+ port: parseInt(process.env.REDIS_PORT || "6379", 10),
+ ...(process.env.REDIS_PASSWORD ? { password: process.env.REDIS_PASSWORD } : {}),
+ commandTimeout: REDIS_COMMAND_TIMEOUT_MS,
+ maxRetriesPerRequest: 3,
+ retryStrategy: (times: number) => Math.min(times * 200, 5000),
+ });
+
+ client.on("error", (error: Error) => logger.error(error, `Redis client error (${label})`));
+ client.on("connect", () => logger.info(`Redis connected (${label})`));
+
+ return client;
+}
+
+// Shared request/response client for bot detection (anomaly scoring + stats) and
+// general use. Distinct from the BullMQ connections used by the uptime service
+// (those have their own settings).
+export const redis = createRedisClient("main");
-redis.on("error", (error: Error) => logger.error(error, "Redis client error"));
-redis.on("connect", () => logger.info("Redis connected"));
+// Dedicated connection for per-event session resolution. Isolated from `redis`
+// so a session GET/SET is never head-of-line blocked behind the anomaly scorer's
+// Lua call β€” that blocking timed sessions out, and a timed-out session forced the
+// fallback path, which used to fracture one visitor into multiple sessions.
+export const sessionRedis = createRedisClient("session");
// Atomic get-or-create with a sliding TTL. If the key exists we refresh its TTL
// and return the stored id; otherwise we store the candidate and return it. Run
// as a single Lua script so concurrent workers can never create duplicate
// sessions for the same (siteId, userId). defineCommand uses EVALSHA with an
// EVAL fallback, so the script body is only sent over the wire once.
-redis.defineCommand("sessionGetOrCreate", {
+sessionRedis.defineCommand("sessionGetOrCreate", {
numberOfKeys: 1,
lua: `
local existing = redis.call('GET', KEYS[1])
@@ -47,7 +64,7 @@ interface SessionRedis extends Redis {
* (re)setting its TTL to `ttlMs`. Atomic across all workers.
*/
export function sessionGetOrCreate(key: string, candidateId: string, ttlMs: number): Promise<string> {
- return (redis as SessionRedis).sessionGetOrCreate(key, candidateId, ttlMs);
+ return (sessionRedis as SessionRedis).sessionGetOrCreate(key, candidateId, ttlMs);
}
// Rolling-window counters for the bot anomaly scorer. Each counter is a sorted
diff --git a/server/src/services/sessions/sessionsService.test.ts b/server/src/services/sessions/sessionsService.test.ts
index 6ee5344a6..f4b63db67 100644
--- a/server/src/services/sessions/sessionsService.test.ts
+++ b/server/src/services/sessions/sessionsService.test.ts
@@ -6,7 +6,7 @@ const mocks = vi.hoisted(() => ({
}));
vi.mock("../../db/redis/redis.js", () => ({
- redis: { quit: mocks.quit },
+ sessionRedis: { quit: mocks.quit },
sessionGetOrCreate: (...args: unknown[]) => mocks.sessionGetOrCreate(...args),
}));
@@ -58,16 +58,29 @@ describe("SessionsService (Redis-backed)", () => {
mocks.sessionGetOrCreate.mockRejectedValue(new Error("redis down"));
const first = await service.updateSession({ userId: "user-b", siteId: 7 });
- // Same 30-minute bucket β†’ same fallback id, so events still share a session.
+ // Still within the sliding window β†’ same fallback id, so events stay grouped.
vi.setSystemTime(new Date(Date.now() + SESSION_TTL_MS - 1000));
const second = await service.updateSession({ userId: "user-b", siteId: 7 });
expect(first.sessionId).toBe(second.sessionId);
- expect(first.sessionId).toContain("7");
- expect(first.sessionId).toContain("user-b");
+ expect(typeof first.sessionId).toBe("string");
+ expect(first.sessionId.length).toBeGreaterThan(0);
});
- it("rotates the fallback id once the time bucket advances", async () => {
+ it("reuses the real Redis session id when a later command blips, instead of splitting", async () => {
+ // The core regression guard: an intermittent Redis failure must not fracture
+ // a visitor into multiple sessions. The blip should inherit the real id.
+ mocks.sessionGetOrCreate.mockResolvedValueOnce("sess-real");
+ const ok = await service.updateSession({ userId: "user-d", siteId: 9 });
+
+ mocks.sessionGetOrCreate.mockRejectedValueOnce(new Error("redis blip"));
+ const blip = await service.updateSession({ userId: "user-d", siteId: 9 });
+
+ expect(ok.sessionId).toBe("sess-real");
+ expect(blip.sessionId).toBe("sess-real");
+ });
+
+ it("rotates the fallback id once the sliding window lapses", async () => {
mocks.sessionGetOrCreate.mockRejectedValue(new Error("redis down"));
const first = await service.updateSession({ userId: "user-c", siteId: 7 });
diff --git a/server/src/services/sessions/sessionsService.ts b/server/src/services/sessions/sessionsService.ts
index 85847026d..331c98d57 100644
--- a/server/src/services/sessions/sessionsService.ts
+++ b/server/src/services/sessions/sessionsService.ts
@@ -1,14 +1,29 @@
import { nanoid } from "nanoid";
import { createServiceLogger } from "../../lib/logger/logger.js";
-import { redis, sessionGetOrCreate } from "../../db/redis/redis.js";
+import { sessionRedis, sessionGetOrCreate } from "../../db/redis/redis.js";
// Sessions expire after this much inactivity. Redis refreshes the TTL on every
// event (sliding window) and evicts the key automatically once it lapses β€” there
// is no table to scan and no cleanup cron to run.
const SESSION_TTL_MS = 30 * 60 * 1000;
+// Bounded in-process mirror of the last session id Redis handed back for each
+// (siteId, userId). It exists purely so a transient Redis failure can reuse the
+// user's *real* session id instead of inventing a divergent one β€” otherwise a
+// single timed-out command fractures one visitor into multiple sessions (events
+// that reached Redis keep the real id while the failed event mints a new one).
+// Entries slide on every touch and the map is capped LRU-style, so memory stays
+// bounded by the active-user count.
+const FALLBACK_CACHE_MAX = 50_000;
+
+interface CachedSession {
+ sessionId: string;
+ expiresAt: number;
+}
+
export class SessionsService {
private logger = createServiceLogger("sessions");
+ private fallbackCache = new Map<string, CachedSession>();
private getSessionKey(userId: string, siteId: number): string {
return `session:${siteId}:${userId}`;
@@ -16,7 +31,8 @@ export class SessionsService {
/**
* Get the active session id for a (userId, siteId) pair, creating one if none
- * exists, and refresh its sliding 30-minute TTL. Backed entirely by Redis.
+ * exists, and refresh its sliding 30-minute TTL. Backed by Redis, with an
+ * in-process fallback so a Redis blip never drops β€” or splits β€” a session.
*/
async updateSession({ userId, siteId }: { userId: string; siteId: number }): Promise<{ sessionId: string }> {
const key = this.getSessionKey(userId, siteId);
@@ -24,25 +40,47 @@ export class SessionsService {
try {
const sessionId = await sessionGetOrCreate(key, candidate, SESSION_TTL_MS);
+ this.rememberSession(key, sessionId);
return { sessionId };
} catch (error) {
- // A Redis blip must never drop ingestion. Fall back to a deterministic,
- // window-stable id so events in the same 30-minute bucket still share a
- // session until Redis recovers.
- this.logger.error(error as Error, "Redis session lookup failed; using fallback session id");
- return { sessionId: this.fallbackSessionId(userId, siteId) };
+ // A Redis blip must never drop ingestion β€” and must never split a session.
+ // Reuse the last id we handed this user (kept alive in-process on a sliding
+ // window) so their events stay glued together until Redis recovers; only
+ // mint a fresh id if we've genuinely never seen them on this worker.
+ this.logger.error(error as Error, "Redis session lookup failed; using in-process fallback session id");
+ return { sessionId: this.fallbackSessionId(key, candidate) };
+ }
+ }
+
+ /** Cache the resolved session id with a fresh sliding expiry, LRU-bounded. */
+ private rememberSession(key: string, sessionId: string): void {
+ // Re-insert to mark as most-recently-used (Map preserves insertion order).
+ this.fallbackCache.delete(key);
+ this.fallbackCache.set(key, { sessionId, expiresAt: Date.now() + SESSION_TTL_MS });
+ if (this.fallbackCache.size > FALLBACK_CACHE_MAX) {
+ const oldest = this.fallbackCache.keys().next().value;
+ if (oldest !== undefined) this.fallbackCache.delete(oldest);
}
}
- private fallbackSessionId(userId: string, siteId: number): string {
- const bucket = Math.floor(Date.now() / SESSION_TTL_MS);
- return `f_${siteId}_${userId}_${bucket}`;
+ /**
+ * Resolve a session id without Redis. Reuses the last id seen for this user if
+ * it's still within the sliding window (refreshing it), otherwise adopts
+ * `candidate` as a new session. The result is stored back so subsequent events
+ * during the same outage keep the same id β€” a per-worker stand-in for Redis
+ * that stays stable across the 30-minute boundary instead of resetting on it.
+ */
+ private fallbackSessionId(key: string, candidate: string): string {
+ const cached = this.fallbackCache.get(key);
+ const sessionId = cached && cached.expiresAt > Date.now() ? cached.sessionId : candidate;
+ this.rememberSession(key, sessionId);
+ return sessionId;
}
- /** Close the Redis connection during graceful shutdown. */
+ /** Close the dedicated session Redis connection during graceful shutdown. */
async close(): Promise<void> {
try {
- await redis.quit();
+ await sessionRedis.quit();
} catch (error) {
this.logger.error(error as Error, "Error closing Redis connection");
}