OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): resolve the session from both salt windows in one Redis read
Carl-Gerhard Lindesvärdfdbcf9af904b
diff --git a/packages/core/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.tsindex f65b28a44..33ff6d3ba 100644--- a/packages/core/src/buffers/session-buffer.test.ts+++ b/packages/core/src/buffers/session-buffer.test.ts@@ -98,6 +98,25 @@ describe('SessionBuffer', () => {expect(parsed.device_id).toBe(deviceId);});+ it('getExistingSessions reads several devices in order, null when absent', async () => {+ await sessionBuffer.ingest(makePayload({ deviceId: 'device-A' }));+ await sessionBuffer.ingest(+ makePayload({ deviceId: 'device-C', sessionId: 'session-C' })+ );++ const sessions = await sessionBuffer.getExistingSessions(projectId, [+ 'device-A',+ 'device-B',+ 'device-C',+ ]);++ expect(sessions.map((session) => session?.id ?? null)).toEqual([+ sessionId,+ null,+ 'session-C',+ ]);+ });+it('registers the device in the wallclock sorted set and the projects set', async () => {await sessionBuffer.ingest(makePayload());diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.tsindex 02ff81e85..c3b94f74e 100644--- a/packages/core/src/buffers/session-buffer.ts+++ b/packages/core/src/buffers/session-buffer.ts@@ -130,6 +130,22 @@ export class SessionBuffer extends BaseBuffer {return hit ? getSafeJson<IClickhouseSession>(hit) : null;}+ /** The live sessions of several devices in one MGET, in the given order. */+ public async getExistingSessions(+ projectId: string,+ deviceIds: string[]+ ): Promise<(IClickhouseSession | null)[]> {+ if (deviceIds.length === 0) {+ return [];+ }+ const hits = await this.redis.mget(+ deviceIds.map((deviceId) => sessionKey(projectId, deviceId))+ );+ return hits.map((hit) =>+ hit ? getSafeJson<IClickhouseSession>(hit) : null+ );+ }+/*** Ingest one event into the session lifecycle. Returns the action taken so* the caller can drive session_start / session_end event emission —diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.tsindex 9ed251dd5..d03b9b50a 100644--- a/packages/core/src/modules/ingest/ingest.service.test.ts+++ b/packages/core/src/modules/ingest/ingest.service.test.ts@@ -220,14 +220,16 @@ describe('handleReplay', () => {const PROJECT_ID = 'proj-kafka';const DEVICE_ID = 'device-kafka-1';-const getExistingSession = mock(async () => null);+const getExistingSessions = mock(+ async (_projectId: string, deviceIds: string[]) => deviceIds.map(() => null)+);const transport = (produceIncomingEvent: ReturnType<typeof mock>): Parameters<typeof ingestTrack>[1] =>({buffers: {- session: { getExistingSession },+ session: { getExistingSessions },replay: { add: mock(async () => undefined) },group: { add: mock(async () => undefined) },},@@ -246,7 +248,7 @@ const request = (body: ITrackHandlerPayload) => ({describe('ingestTrack — events transport', () => {beforeEach(() => {- getExistingSession.mockClear();+ getExistingSessions.mockClear();identifyProfile.mockClear();});diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex f5d99cc38..ef894b980 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -750,16 +750,10 @@ export async function fetchDeviceIdentity(});try {- const [current, previous] = await Promise.all([- buffers.session.getExistingSession({- projectId,- deviceId: currentDeviceId,- }),- buffers.session.getExistingSession({- projectId,- deviceId: previousDeviceId,- }),- ]);+ const [current, previous] = await buffers.session.getExistingSessions(+ projectId,+ [currentDeviceId, previousDeviceId]+ );// Blob has no TTL — only treat the session as "current" if its last// event is within the idle window. Otherwise the SDK should ask thediff --git a/packages/core/src/modules/ingest/legacy-event.test.ts b/packages/core/src/modules/ingest/legacy-event.test.tsindex ff1358e28..82cda43a0 100644--- a/packages/core/src/modules/ingest/legacy-event.test.ts+++ b/packages/core/src/modules/ingest/legacy-event.test.ts@@ -79,9 +79,12 @@ const produceIncomingEvent = async (};// The legacy route resolves a device id, which is the only buffer read on-// this path (`getExistingSession`, once per salt candidate).+// this path (`getExistingSessions`, one read for both salt candidates).const buffers = {- session: { getExistingSession: async () => null },+ session: {+ getExistingSessions: async (_projectId: string, deviceIds: string[]) =>+ deviceIds.map(() => null),+ },replay: {},group: {},} as unknown as Buffers;diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.tsindex eca518c82..5c616643d 100644--- a/packages/core/src/modules/ingest/src/device-id.test.ts+++ b/packages/core/src/modules/ingest/src/device-id.test.ts@@ -46,10 +46,13 @@ const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>}) as unknown as IClickhouseSession;function stubBuffer(session: IClickhouseSession | null) {- const getExistingSession = mock(async () => session);+ const getExistingSessions = mock(+ async (_projectId: string, deviceIds: string[]) =>+ deviceIds.map(() => session)+ );return {- getExistingSession,- buffer: { getExistingSession } as unknown as SessionBufferReader,+ getExistingSessions,+ buffer: { getExistingSessions } satisfies SessionBufferReader,};}@@ -59,7 +62,7 @@ afterEach(() => {describe('getDeviceId — session resolution', () => {it('mints a deterministic, stable id when no session exists', async () => {- const { getExistingSession, buffer } = stubBuffer(null);+ const { getExistingSessions, buffer } = stubBuffer(null);const a = await getDeviceId({...BASE,@@ -75,7 +78,7 @@ describe('getDeviceId — session resolution', () => {expect(a.sessionId).toBeTruthy();expect(b.sessionId).toBe(a.sessionId); // same window → same idexpect(a.deviceId).toBe('cookie-abc');- expect(getExistingSession).toHaveBeenCalled();+ expect(getExistingSessions).toHaveBeenCalled();});it('reuses the live session id when within the idle window', async () => {@@ -106,8 +109,8 @@ describe('getDeviceId — session resolution', () => {expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one});- it('reads the store once for an override (no redundant previous lookup)', async () => {- const { getExistingSession, buffer } = stubBuffer(null);+ it('reads only the override device (no redundant previous lookup)', async () => {+ const { getExistingSessions, buffer } = stubBuffer(null);await getDeviceId({...BASE,@@ -115,23 +118,17 @@ describe('getDeviceId — session resolution', () => {sessionBuffer: buffer,});- expect(getExistingSession).toHaveBeenCalledTimes(1);- expect(getExistingSession).toHaveBeenCalledWith({- projectId: 'proj-1',- deviceId: 'cookie-abc',- });+ expect(getExistingSessions).toHaveBeenCalledTimes(1);+ expect(getExistingSessions).toHaveBeenCalledWith('proj-1', ['cookie-abc']);});- it('checks both current and previous salt windows for internal ids', async () => {- const { getExistingSession, buffer } = stubBuffer(null);+ it('checks both salt windows for internal ids in one read', async () => {+ const { getExistingSessions, buffer } = stubBuffer(null);await getDeviceId({ ...BASE, sessionBuffer: buffer }); // no override → IP+UA hashing- expect(getExistingSession).toHaveBeenCalledTimes(2);- const deviceIds = getExistingSession.mock.calls.map((call) => {- const [args] = call as unknown as [{ deviceId?: string }];- return args.deviceId ?? '';- });+ expect(getExistingSessions).toHaveBeenCalledTimes(1);+ const [, deviceIds] = getExistingSessions.mock.calls[0]!;expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes});@@ -145,7 +142,7 @@ describe('getDeviceId — session resolution', () => {it('mints the SAME deterministic id whether the session read misses or rejects', async () => {const { buffer: emptyStore } = stubBuffer(null);const rejecting = {- getExistingSession: mock(() =>+ getExistingSessions: mock(() =>Promise.reject(new Error("Stream isn't writeable and enableOfflineQueue options is false"@@ -173,7 +170,7 @@ describe('getDeviceId — session resolution', () => {it('logs a failed session read through the caller logger, not console', async () => {const { error, logger } = stubLogger();const failing = {- getExistingSession: mock(() => {+ getExistingSessions: mock(() => {throw new Error('session store unavailable');}),} as unknown as SessionBufferReader;diff --git a/packages/core/src/modules/ingest/src/device-id.ts b/packages/core/src/modules/ingest/src/device-id.tsindex 795ca9325..cd49e2e02 100644--- a/packages/core/src/modules/ingest/src/device-id.ts+++ b/packages/core/src/modules/ingest/src/device-id.ts@@ -10,7 +10,7 @@ import type { IClickhouseSession } from '../../session/session.service';import { convertClickhouseDateToJs } from '../../session/src/dates';/** Only the read this module needs, so a caller may pass a narrower stub. */-export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSession'>;+export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSessions'>;/** Only the level this module logs at. The inputs here are an IP and a salt,* so it must be the redacting pino logger and never `console`. */@@ -139,10 +139,9 @@ async function getInfoFromSession({// Reading the live blob is the source of truth for an active session — it's// what keeps a visit on one id across page reloads and salt rotation. Don't// drop this read to "save" a lookup or sessions split at the bucket boundary.- const sessions = await Promise.all(- candidates.map((deviceId) =>- sessionBuffer.getExistingSession({ projectId, deviceId })- )+ const sessions = await sessionBuffer.getExistingSessions(+ projectId,+ candidates);for (const [index, session] of sessions.entries()) {