Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(api): resolve the session from both salt windows in one Redis read

Carl-Gerhard Lindesvärdfdbcf9af904b
diff --git a/packages/core/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.ts
index f65b28a44..33ff6d3ba 100644
--- a/packages/core/src/buffers/session-buffer.test.ts
+++ b/packages/core/src/buffers/session-buffer.test.ts
@@ -98,6 +98,25 @@ describe('SessionBuffer', () => {
expect(parsed.device_id).toBe(deviceId);
});
+ it('getExistingSessions reads several devices in order, null when absent', async () => {
+ await sessionBuffer.ingest(makePayload({ deviceId: 'device-A' }));
+ await sessionBuffer.ingest(
+ makePayload({ deviceId: 'device-C', sessionId: 'session-C' })
+ );
+
+ const sessions = await sessionBuffer.getExistingSessions(projectId, [
+ 'device-A',
+ 'device-B',
+ 'device-C',
+ ]);
+
+ expect(sessions.map((session) => session?.id ?? null)).toEqual([
+ sessionId,
+ null,
+ 'session-C',
+ ]);
+ });
+
it('registers the device in the wallclock sorted set and the projects set', async () => {
await sessionBuffer.ingest(makePayload());
diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.ts
index 02ff81e85..c3b94f74e 100644
--- a/packages/core/src/buffers/session-buffer.ts
+++ b/packages/core/src/buffers/session-buffer.ts
@@ -130,6 +130,22 @@ export class SessionBuffer extends BaseBuffer {
return hit ? getSafeJson<IClickhouseSession>(hit) : null;
}
+ /** The live sessions of several devices in one MGET, in the given order. */
+ public async getExistingSessions(
+ projectId: string,
+ deviceIds: string[]
+ ): Promise<(IClickhouseSession | null)[]> {
+ if (deviceIds.length === 0) {
+ return [];
+ }
+ const hits = await this.redis.mget(
+ deviceIds.map((deviceId) => sessionKey(projectId, deviceId))
+ );
+ return hits.map((hit) =>
+ hit ? getSafeJson<IClickhouseSession>(hit) : null
+ );
+ }
+
/**
* Ingest one event into the session lifecycle. Returns the action taken so
* the caller can drive session_start / session_end event emission —
diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.ts
index 9ed251dd5..d03b9b50a 100644
--- a/packages/core/src/modules/ingest/ingest.service.test.ts
+++ b/packages/core/src/modules/ingest/ingest.service.test.ts
@@ -220,14 +220,16 @@ describe('handleReplay', () => {
const PROJECT_ID = 'proj-kafka';
const DEVICE_ID = 'device-kafka-1';
-const getExistingSession = mock(async () => null);
+const getExistingSessions = mock(
+ async (_projectId: string, deviceIds: string[]) => deviceIds.map(() => null)
+);
const transport = (
produceIncomingEvent: ReturnType<typeof mock>
): Parameters<typeof ingestTrack>[1] =>
({
buffers: {
- session: { getExistingSession },
+ session: { getExistingSessions },
replay: { add: mock(async () => undefined) },
group: { add: mock(async () => undefined) },
},
@@ -246,7 +248,7 @@ const request = (body: ITrackHandlerPayload) => ({
describe('ingestTrack — events transport', () => {
beforeEach(() => {
- getExistingSession.mockClear();
+ getExistingSessions.mockClear();
identifyProfile.mockClear();
});
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
index f5d99cc38..ef894b980 100644
--- a/packages/core/src/modules/ingest/ingest.service.ts
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -750,16 +750,10 @@ export async function fetchDeviceIdentity(
});
try {
- const [current, previous] = await Promise.all([
- buffers.session.getExistingSession({
- projectId,
- deviceId: currentDeviceId,
- }),
- buffers.session.getExistingSession({
- projectId,
- deviceId: previousDeviceId,
- }),
- ]);
+ const [current, previous] = await buffers.session.getExistingSessions(
+ projectId,
+ [currentDeviceId, previousDeviceId]
+ );
// Blob has no TTL — only treat the session as "current" if its last
// event is within the idle window. Otherwise the SDK should ask the
diff --git a/packages/core/src/modules/ingest/legacy-event.test.ts b/packages/core/src/modules/ingest/legacy-event.test.ts
index ff1358e28..82cda43a0 100644
--- a/packages/core/src/modules/ingest/legacy-event.test.ts
+++ b/packages/core/src/modules/ingest/legacy-event.test.ts
@@ -79,9 +79,12 @@ const produceIncomingEvent = async (
};
// The legacy route resolves a device id, which is the only buffer read on
-// this path (`getExistingSession`, once per salt candidate).
+// this path (`getExistingSessions`, one read for both salt candidates).
const buffers = {
- session: { getExistingSession: async () => null },
+ session: {
+ getExistingSessions: async (_projectId: string, deviceIds: string[]) =>
+ deviceIds.map(() => null),
+ },
replay: {},
group: {},
} as unknown as Buffers;
diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.ts
index eca518c82..5c616643d 100644
--- a/packages/core/src/modules/ingest/src/device-id.test.ts
+++ b/packages/core/src/modules/ingest/src/device-id.test.ts
@@ -46,10 +46,13 @@ const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>
}) as unknown as IClickhouseSession;
function stubBuffer(session: IClickhouseSession | null) {
- const getExistingSession = mock(async () => session);
+ const getExistingSessions = mock(
+ async (_projectId: string, deviceIds: string[]) =>
+ deviceIds.map(() => session)
+ );
return {
- getExistingSession,
- buffer: { getExistingSession } as unknown as SessionBufferReader,
+ getExistingSessions,
+ buffer: { getExistingSessions } satisfies SessionBufferReader,
};
}
@@ -59,7 +62,7 @@ afterEach(() => {
describe('getDeviceId — session resolution', () => {
it('mints a deterministic, stable id when no session exists', async () => {
- const { getExistingSession, buffer } = stubBuffer(null);
+ const { getExistingSessions, buffer } = stubBuffer(null);
const a = await getDeviceId({
...BASE,
@@ -75,7 +78,7 @@ describe('getDeviceId — session resolution', () => {
expect(a.sessionId).toBeTruthy();
expect(b.sessionId).toBe(a.sessionId); // same window → same id
expect(a.deviceId).toBe('cookie-abc');
- expect(getExistingSession).toHaveBeenCalled();
+ expect(getExistingSessions).toHaveBeenCalled();
});
it('reuses the live session id when within the idle window', async () => {
@@ -106,8 +109,8 @@ describe('getDeviceId — session resolution', () => {
expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one
});
- it('reads the store once for an override (no redundant previous lookup)', async () => {
- const { getExistingSession, buffer } = stubBuffer(null);
+ it('reads only the override device (no redundant previous lookup)', async () => {
+ const { getExistingSessions, buffer } = stubBuffer(null);
await getDeviceId({
...BASE,
@@ -115,23 +118,17 @@ describe('getDeviceId — session resolution', () => {
sessionBuffer: buffer,
});
- expect(getExistingSession).toHaveBeenCalledTimes(1);
- expect(getExistingSession).toHaveBeenCalledWith({
- projectId: 'proj-1',
- deviceId: 'cookie-abc',
- });
+ expect(getExistingSessions).toHaveBeenCalledTimes(1);
+ expect(getExistingSessions).toHaveBeenCalledWith('proj-1', ['cookie-abc']);
});
- it('checks both current and previous salt windows for internal ids', async () => {
- const { getExistingSession, buffer } = stubBuffer(null);
+ it('checks both salt windows for internal ids in one read', async () => {
+ const { getExistingSessions, buffer } = stubBuffer(null);
await getDeviceId({ ...BASE, sessionBuffer: buffer }); // no override → IP+UA hashing
- expect(getExistingSession).toHaveBeenCalledTimes(2);
- const deviceIds = getExistingSession.mock.calls.map((call) => {
- const [args] = call as unknown as [{ deviceId?: string }];
- return args.deviceId ?? '';
- });
+ expect(getExistingSessions).toHaveBeenCalledTimes(1);
+ const [, deviceIds] = getExistingSessions.mock.calls[0]!;
expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes
});
@@ -145,7 +142,7 @@ describe('getDeviceId — session resolution', () => {
it('mints the SAME deterministic id whether the session read misses or rejects', async () => {
const { buffer: emptyStore } = stubBuffer(null);
const rejecting = {
- getExistingSession: mock(() =>
+ getExistingSessions: mock(() =>
Promise.reject(
new Error(
"Stream isn't writeable and enableOfflineQueue options is false"
@@ -173,7 +170,7 @@ describe('getDeviceId — session resolution', () => {
it('logs a failed session read through the caller logger, not console', async () => {
const { error, logger } = stubLogger();
const failing = {
- getExistingSession: mock(() => {
+ getExistingSessions: mock(() => {
throw new Error('session store unavailable');
}),
} as unknown as SessionBufferReader;
diff --git a/packages/core/src/modules/ingest/src/device-id.ts b/packages/core/src/modules/ingest/src/device-id.ts
index 795ca9325..cd49e2e02 100644
--- a/packages/core/src/modules/ingest/src/device-id.ts
+++ b/packages/core/src/modules/ingest/src/device-id.ts
@@ -10,7 +10,7 @@ import type { IClickhouseSession } from '../../session/session.service';
import { convertClickhouseDateToJs } from '../../session/src/dates';
/** Only the read this module needs, so a caller may pass a narrower stub. */
-export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSession'>;
+export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSessions'>;
/** Only the level this module logs at. The inputs here are an IP and a salt,
* so it must be the redacting pino logger and never `console`. */
@@ -139,10 +139,9 @@ async function getInfoFromSession({
// Reading the live blob is the source of truth for an active session — it's
// what keeps a visit on one id across page reloads and salt rotation. Don't
// drop this read to "save" a lookup or sessions split at the bucket boundary.
- const sessions = await Promise.all(
- candidates.map((deviceId) =>
- sessionBuffer.getExistingSession({ projectId, deviceId })
- )
+ const sessions = await sessionBuffer.getExistingSessions(
+ projectId,
+ candidates
);
for (const [index, session] of sessions.entries()) {