Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(ingest): session profile pointers no longer accumulate in Redis forever

Carl-Gerhard Lindesvärdf32587281f32
diff --git a/packages/core/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.ts
index 33ff6d3ba..38205c492 100644
--- a/packages/core/src/buffers/session-buffer.test.ts
+++ b/packages/core/src/buffers/session-buffer.test.ts
@@ -126,15 +126,50 @@ describe('SessionBuffer', () => {
expect(await redis.sismember('session:projects', projectId)).toBe(1);
});
- it('writes no TTL on the session blob or profile pointer', async () => {
+ it('writes no TTL on the session blob, but one on the profile pointer', async () => {
await sessionBuffer.ingest(
makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
);
- // Blob and profile pointer must survive arbitrarily long reaper outages,
- // so neither carries a Redis TTL.
+ // The blob must survive arbitrarily long reaper outages. The pointer is
+ // in no index, so its TTL is what collects it when cleanup() misses it.
expect(await redis.ttl(`session:${projectId}:device-Y`)).toBe(-1);
- expect(await redis.ttl(`session:profile:${projectId}:profile-X`)).toBe(-1);
+ const pointerTtl = await redis.pttl(
+ `session:profile:${projectId}:profile-X`
+ );
+ expect(pointerTtl).toBeGreaterThan(30 * 60 * 1000);
+ expect(pointerTtl).toBeLessThanOrEqual(60 * 60 * 1000);
+ });
+
+ it('drops the previous profile pointer when a session is re-identified', async () => {
+ await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));
+ await sessionBuffer.ingest(
+ makePayload({
+ profileId: 'profile-B',
+ createdAt: new Date(Date.now() + 1000),
+ })
+ );
+
+ expect(
+ await redis.get(`session:profile:${projectId}:profile-A`)
+ ).toBeNull();
+ expect(await redis.get(`session:profile:${projectId}:profile-B`)).toBe(
+ deviceId
+ );
+ });
+
+ it('keeps the profile pointer when an extend repeats the same profile', async () => {
+ await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));
+ await sessionBuffer.ingest(
+ makePayload({
+ profileId: 'profile-A',
+ createdAt: new Date(Date.now() + 1000),
+ })
+ );
+
+ expect(await redis.get(`session:profile:${projectId}:profile-A`)).toBe(
+ deviceId
+ );
});
it('writes profile→device pointer when profile_id differs from device_id', async () => {
diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.ts
index dc097c9f7..ffd84356e 100644
--- a/packages/core/src/buffers/session-buffer.ts
+++ b/packages/core/src/buffers/session-buffer.ts
@@ -10,6 +10,11 @@ import { defineRedisScript } from './redis-script';
// 30min of idle in event-time → session ends.
const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;
+// The profile pointer is refreshed on every persist, so it only has to outlive
+// the idle timeout plus the reaper's lag. It sits in no index, so a TTL is the
+// only thing that collects a pointer `cleanup()` never gets handed.
+const PROFILE_POINTER_TTL_FACTOR = 2;
+
/**
* Idle window for a session (boundary detection + the reaper's default
* deadman). Overridable through SESSION_TIMEOUT_MS so the E2E harness can
@@ -90,6 +95,8 @@ export class SessionBuffer extends BaseBuffer {
private readonly squashEnabled = this.deps.config.buffers.session.squash;
private readonly idleTimeoutMs = resolveSessionTimeoutMs(this.deps.config);
+ private readonly profilePointerTtlMs =
+ this.idleTimeoutMs * PROFILE_POINTER_TTL_FACTOR;
private readonly redisKey = 'session-buffer';
private readonly redis: Redis;
@@ -174,7 +181,7 @@ export class SessionBuffer extends BaseBuffer {
if (existing && !isBoundary) {
const { current, chRows } = this.extendSession(existing, payload);
- await this.persist(current, chRows);
+ await this.persist(current, chRows, existing);
return { kind: 'extend', current };
}
@@ -200,7 +207,10 @@ export class SessionBuffer extends BaseBuffer {
*
* Blobs have no Redis TTL, so this is the only path that removes them. The
* daily vacuum cron is a backstop for when it fails (worker crash, network
- * partition).
+ * partition). The profile pointer additionally expires on its own: it is
+ * indexed nowhere, so a pointer this call is not handed (a boundary that
+ * gated the script out, a session re-identified before it closed) would
+ * otherwise stay forever.
*/
async cleanup({
projectId,
@@ -367,27 +377,42 @@ export class SessionBuffer extends BaseBuffer {
* profile index + ClickHouse buffer rows. Append only; the cron drains the
* buffer, so ingest never waits on a ClickHouse insert.
*
- * No TTLs on the blob or profile index — they are removed exclusively by
- * `cleanup()` after `session_end` emission. This guarantees the reaper
- * can always find a session's state regardless of how long the worker
- * has been down.
+ * No TTL on the blob — it is removed exclusively by `cleanup()` after
+ * `session_end` emission, so the reaper can always find a session's state
+ * regardless of how long the worker has been down. The reaper never goes
+ * through the profile pointer, so that one may expire; the only reader is
+ * server-side ingest looking for a live client session to ride along on.
*/
private async persist(
current: IClickhouseSession,
- chRows: IClickhouseSession[]
+ chRows: IClickhouseSession[],
+ previous?: IClickhouseSession
) {
const projectId = current.project_id;
const deviceId = current.device_id;
const hasProfilePointer =
current.profile_id && current.profile_id !== current.device_id;
+ const previousProfileId = previous?.profile_id;
+ const hasStalePointer =
+ previousProfileId &&
+ previousProfileId !== deviceId &&
+ previousProfileId !== current.profile_id;
const multi = this.redis
.multi()
.set(sessionKey(projectId, deviceId), JSON.stringify(current))
.zadd(wallclockSetKey(projectId), Date.now(), deviceId)
.sadd(PROJECTS_SET_KEY, projectId);
+ if (hasStalePointer) {
+ multi.del(profileIndexKey(projectId, previousProfileId));
+ }
if (hasProfilePointer) {
- multi.set(profileIndexKey(projectId, current.profile_id), deviceId);
+ multi.set(
+ profileIndexKey(projectId, current.profile_id),
+ deviceId,
+ 'PX',
+ this.profilePointerTtlMs
+ );
}
multi.rpush(this.redisKey, ...chRows.map((row) => JSON.stringify(row)));