OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(ingest): session profile pointers no longer accumulate in Redis forever
Carl-Gerhard Lindesvärdf32587281f32
diff --git a/packages/core/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.tsindex 33ff6d3ba..38205c492 100644--- a/packages/core/src/buffers/session-buffer.test.ts+++ b/packages/core/src/buffers/session-buffer.test.ts@@ -126,15 +126,50 @@ describe('SessionBuffer', () => {expect(await redis.sismember('session:projects', projectId)).toBe(1);});- it('writes no TTL on the session blob or profile pointer', async () => {+ it('writes no TTL on the session blob, but one on the profile pointer', async () => {await sessionBuffer.ingest(makePayload({ profileId: 'profile-X', deviceId: 'device-Y' }));- // Blob and profile pointer must survive arbitrarily long reaper outages,- // so neither carries a Redis TTL.+ // The blob must survive arbitrarily long reaper outages. The pointer is+ // in no index, so its TTL is what collects it when cleanup() misses it.expect(await redis.ttl(`session:${projectId}:device-Y`)).toBe(-1);- expect(await redis.ttl(`session:profile:${projectId}:profile-X`)).toBe(-1);+ const pointerTtl = await redis.pttl(+ `session:profile:${projectId}:profile-X`+ );+ expect(pointerTtl).toBeGreaterThan(30 * 60 * 1000);+ expect(pointerTtl).toBeLessThanOrEqual(60 * 60 * 1000);+ });++ it('drops the previous profile pointer when a session is re-identified', async () => {+ await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));+ await sessionBuffer.ingest(+ makePayload({+ profileId: 'profile-B',+ createdAt: new Date(Date.now() + 1000),+ })+ );++ expect(+ await redis.get(`session:profile:${projectId}:profile-A`)+ ).toBeNull();+ expect(await redis.get(`session:profile:${projectId}:profile-B`)).toBe(+ deviceId+ );+ });++ it('keeps the profile pointer when an extend repeats the same profile', async () => {+ await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));+ await sessionBuffer.ingest(+ makePayload({+ profileId: 'profile-A',+ createdAt: new Date(Date.now() + 1000),+ })+ );++ expect(await redis.get(`session:profile:${projectId}:profile-A`)).toBe(+ deviceId+ );});it('writes profile→device pointer when profile_id differs from device_id', async () => {diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.tsindex dc097c9f7..ffd84356e 100644--- a/packages/core/src/buffers/session-buffer.ts+++ b/packages/core/src/buffers/session-buffer.ts@@ -10,6 +10,11 @@ import { defineRedisScript } from './redis-script';// 30min of idle in event-time → session ends.const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;+// The profile pointer is refreshed on every persist, so it only has to outlive+// the idle timeout plus the reaper's lag. It sits in no index, so a TTL is the+// only thing that collects a pointer `cleanup()` never gets handed.+const PROFILE_POINTER_TTL_FACTOR = 2;+/*** Idle window for a session (boundary detection + the reaper's default* deadman). Overridable through SESSION_TIMEOUT_MS so the E2E harness can@@ -90,6 +95,8 @@ export class SessionBuffer extends BaseBuffer {private readonly squashEnabled = this.deps.config.buffers.session.squash;private readonly idleTimeoutMs = resolveSessionTimeoutMs(this.deps.config);+ private readonly profilePointerTtlMs =+ this.idleTimeoutMs * PROFILE_POINTER_TTL_FACTOR;private readonly redisKey = 'session-buffer';private readonly redis: Redis;@@ -174,7 +181,7 @@ export class SessionBuffer extends BaseBuffer {if (existing && !isBoundary) {const { current, chRows } = this.extendSession(existing, payload);- await this.persist(current, chRows);+ await this.persist(current, chRows, existing);return { kind: 'extend', current };}@@ -200,7 +207,10 @@ export class SessionBuffer extends BaseBuffer {** Blobs have no Redis TTL, so this is the only path that removes them. The* daily vacuum cron is a backstop for when it fails (worker crash, network- * partition).+ * partition). The profile pointer additionally expires on its own: it is+ * indexed nowhere, so a pointer this call is not handed (a boundary that+ * gated the script out, a session re-identified before it closed) would+ * otherwise stay forever.*/async cleanup({projectId,@@ -367,27 +377,42 @@ export class SessionBuffer extends BaseBuffer {* profile index + ClickHouse buffer rows. Append only; the cron drains the* buffer, so ingest never waits on a ClickHouse insert.*- * No TTLs on the blob or profile index — they are removed exclusively by- * `cleanup()` after `session_end` emission. This guarantees the reaper- * can always find a session's state regardless of how long the worker- * has been down.+ * No TTL on the blob — it is removed exclusively by `cleanup()` after+ * `session_end` emission, so the reaper can always find a session's state+ * regardless of how long the worker has been down. The reaper never goes+ * through the profile pointer, so that one may expire; the only reader is+ * server-side ingest looking for a live client session to ride along on.*/private async persist(current: IClickhouseSession,- chRows: IClickhouseSession[]+ chRows: IClickhouseSession[],+ previous?: IClickhouseSession) {const projectId = current.project_id;const deviceId = current.device_id;const hasProfilePointer =current.profile_id && current.profile_id !== current.device_id;+ const previousProfileId = previous?.profile_id;+ const hasStalePointer =+ previousProfileId &&+ previousProfileId !== deviceId &&+ previousProfileId !== current.profile_id;const multi = this.redis.multi().set(sessionKey(projectId, deviceId), JSON.stringify(current)).zadd(wallclockSetKey(projectId), Date.now(), deviceId).sadd(PROJECTS_SET_KEY, projectId);+ if (hasStalePointer) {+ multi.del(profileIndexKey(projectId, previousProfileId));+ }if (hasProfilePointer) {- multi.set(profileIndexKey(projectId, current.profile_id), deviceId);+ multi.set(+ profileIndexKey(projectId, current.profile_id),+ deviceId,+ 'PX',+ this.profilePointerTtlMs+ );}multi.rpush(this.redisKey, ...chRows.map((row) => JSON.stringify(row)));