OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
import { beforeEach, describe, expect, it, mock } from 'bun:test';
import { getRedisCache } from '@openpanel/redis';
import { bufferDepsWithCh } from '../../test/buffer-fixtures';
import { testCoreConfig } from '../../test/config-fixture';
import type { IServiceCreateEventPayload } from '../modules/event/event.service';
import type { IClickhouseSession } from '../modules/session/session.service';
import type { FlushObservation } from './base-buffer';
// The client comes in as `BufferDeps.ch` — no module mock needed for the flush
// path at all. See event-buffer.test.ts's header.
const chInsert = mock(
async (_options: {
table: string;
format: string;
values: IClickhouseSession[];
}): Promise<unknown> => undefined
);
const { SessionBuffer } = await import('./session-buffer');
type SessionBuffer = InstanceType<typeof SessionBuffer>;
const redis = getRedisCache();
const projectId = 'project-1';
const deviceId = 'device-1';
const sessionId = 'session-1';
function makePayload(
overrides: Partial<IServiceCreateEventPayload> = {}
): IServiceCreateEventPayload {
return {
name: 'screen_view',
projectId,
deviceId,
sessionId,
profileId: 'profile-1',
properties: {},
groups: [],
createdAt: new Date(),
duration: 0,
sdkName: 'web',
sdkVersion: '1.0.0',
city: '',
country: '',
region: '',
longitude: 0,
latitude: 0,
path: '/home',
origin: '',
referrer: '',
referrerName: '',
referrerType: '',
os: '',
osVersion: '',
browser: '',
browserVersion: '',
device: '',
brand: '',
model: '',
...overrides,
};
}
beforeEach(async () => {
const keys = [
...(await redis.keys('session*')),
...(await redis.keys('lock:session')),
];
// Spread breaks at ~100k args — del in chunks so old test state can't kill us.
const BATCH = 1000;
for (let i = 0; i < keys.length; i += BATCH) {
await redis.del(...keys.slice(i, i + BATCH));
}
chInsert.mockReset();
chInsert.mockResolvedValue(undefined);
});
// No `afterAll` on purpose: the shared `getRedisCache()` client must NOT be
// quit, because bun runs every file in one process and closing the singleton
// takes it away from the files that run next.
describe('SessionBuffer', () => {
let sessionBuffer: SessionBuffer;
beforeEach(() => {
sessionBuffer = new SessionBuffer(bufferDepsWithCh({ insert: chInsert }));
});
it('opens a new session and stores it at session:{pid}:{did}', async () => {
const result = await sessionBuffer.ingest(makePayload());
expect(result?.kind).toBe('new');
const blob = await redis.get(`session:${projectId}:${deviceId}`);
expect(blob).not.toBeNull();
const parsed = JSON.parse(blob ?? '');
expect(parsed.id).toBe(sessionId);
expect(parsed.project_id).toBe(projectId);
expect(parsed.device_id).toBe(deviceId);
});
it('getExistingSessions reads several devices in order, null when absent', async () => {
await sessionBuffer.ingest(makePayload({ deviceId: 'device-A' }));
await sessionBuffer.ingest(
makePayload({ deviceId: 'device-C', sessionId: 'session-C' })
);
const sessions = await sessionBuffer.getExistingSessions(projectId, [
'device-A',
'device-B',
'device-C',
]);
expect(sessions.map((session) => session?.id ?? null)).toEqual([
sessionId,
null,
'session-C',
]);
});
it('registers the device in the wallclock sorted set and the projects set', async () => {
await sessionBuffer.ingest(makePayload());
expect(
await redis.zscore(`session:wallclock:${projectId}`, deviceId)
).not.toBeNull();
expect(await redis.sismember('session:projects', projectId)).toBe(1);
});
it('writes no TTL on the session blob, but one on the profile pointer', async () => {
await sessionBuffer.ingest(
makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
);
// The blob must survive arbitrarily long reaper outages. The pointer is
// in no index, so its TTL is what collects it when cleanup() misses it.
expect(await redis.ttl(`session:${projectId}:device-Y`)).toBe(-1);
const pointerTtl = await redis.pttl(
`session:profile:${projectId}:profile-X`
);
expect(pointerTtl).toBeGreaterThan(30 * 60 * 1000);
expect(pointerTtl).toBeLessThanOrEqual(60 * 60 * 1000);
});
it('drops the previous profile pointer when a session is re-identified', async () => {
await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));
await sessionBuffer.ingest(
makePayload({
profileId: 'profile-B',
createdAt: new Date(Date.now() + 1000),
})
);
expect(
await redis.get(`session:profile:${projectId}:profile-A`)
).toBeNull();
expect(await redis.get(`session:profile:${projectId}:profile-B`)).toBe(
deviceId
);
});
it('keeps the profile pointer when an extend repeats the same profile', async () => {
await sessionBuffer.ingest(makePayload({ profileId: 'profile-A' }));
await sessionBuffer.ingest(
makePayload({
profileId: 'profile-A',
createdAt: new Date(Date.now() + 1000),
})
);
expect(await redis.get(`session:profile:${projectId}:profile-A`)).toBe(
deviceId
);
});
it('writes profile→device pointer when profile_id differs from device_id', async () => {
await sessionBuffer.ingest(
makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
);
expect(await redis.get(`session:profile:${projectId}:profile-X`)).toBe(
'device-Y'
);
});
it('skips session_start and session_end events', async () => {
const sizeBefore = await sessionBuffer.getBufferSize();
expect(
await sessionBuffer.ingest(makePayload({ name: 'session_start' }))
).toBeNull();
expect(
await sessionBuffer.ingest(makePayload({ name: 'session_end' }))
).toBeNull();
expect(await sessionBuffer.getBufferSize()).toBe(sizeBefore);
});
it('emits sign=-1 + sign=+1 CH rows when extending within timeout', async () => {
const t0 = new Date();
await sessionBuffer.ingest(makePayload({ createdAt: t0 }));
const sizeBefore = await sessionBuffer.getBufferSize();
const result = await sessionBuffer.ingest(
makePayload({ createdAt: new Date(t0.getTime() + 5000) })
);
expect(result?.kind).toBe('extend');
expect(await sessionBuffer.getBufferSize()).toBe(sizeBefore + 2);
});
it('returns kind=boundary when gap > 30min, with closed + current populated', async () => {
const t0 = new Date();
const first = await sessionBuffer.ingest(makePayload({ createdAt: t0 }));
expect(first?.kind).toBe('new');
const second = await sessionBuffer.ingest(
makePayload({
createdAt: new Date(t0.getTime() + 35 * 60 * 1000),
sessionId: 'session-2',
})
);
expect(second?.kind).toBe('boundary');
if (second?.kind === 'boundary') {
expect(second.closed.id).toBe(sessionId);
expect(second.current.id).toBe('session-2');
}
});
it('invariant: one visit keeps one id across extends; a >30min gap splits and closes the first', async () => {
// This ties the layers together against real Redis: ingest() drives the
// state machine, and getExistingSession() is what the API reads on each
// request to resolve the id — they must agree throughout a visit.
const t0 = new Date('2026-06-08T12:00:00.000Z');
const at = (mins: number) => new Date(t0.getTime() + mins * 60_000);
const e1 = await sessionBuffer.ingest(
makePayload({ createdAt: t0, sessionId: 'S1' })
);
expect(e1?.kind).toBe('new');
expect(e1?.current.id).toBe('S1');
// The API's next lookup must see the live session.
expect(
(await sessionBuffer.getExistingSession({ projectId, deviceId }))?.id
).toBe('S1');
// Events within the idle window extend the SAME session.
for (const mins of [10, 20]) {
const ext = await sessionBuffer.ingest(
makePayload({ createdAt: at(mins), sessionId: 'S1' })
);
expect(ext?.kind).toBe('extend');
expect(ext?.current.id).toBe('S1');
}
// A >30min gap: ingest detects the boundary, closes S1, opens S2.
const e4 = await sessionBuffer.ingest(
makePayload({ createdAt: at(55), sessionId: 'S2' })
);
expect(e4?.kind).toBe('boundary');
if (e4?.kind === 'boundary') {
expect(e4.closed.id).toBe('S1');
expect(e4.current.id).toBe('S2');
}
// The slot now holds the new session — the API reads S2 from here on.
expect(
(await sessionBuffer.getExistingSession({ projectId, deviceId }))?.id
).toBe('S2');
});
it('inherits utm_* fields from event.properties.__query', async () => {
await sessionBuffer.ingest(
makePayload({
properties: {
__query: {
utm_medium: 'cpc',
utm_source: 'google',
utm_campaign: 'spring',
},
},
})
);
const blob = JSON.parse(
(await redis.get(`session:${projectId}:${deviceId}`)) ?? ''
);
expect(blob.utm_medium).toBe('cpc');
expect(blob.utm_source).toBe('google');
expect(blob.utm_campaign).toBe('spring');
});
it('out-of-order events: older event extends backwards, never produces negative duration', async () => {
const t0 = new Date();
await sessionBuffer.ingest(makePayload({ createdAt: t0 }));
// Late-arriving event with an older timestamp.
await sessionBuffer.ingest(
makePayload({ createdAt: new Date(t0.getTime() - 30_000) })
);
const blob = JSON.parse(
(await redis.get(`session:${projectId}:${deviceId}`)) ?? ''
);
expect(blob.duration).toBeGreaterThanOrEqual(0);
});
it('cleanup() removes blob, wallclock entry, and profile pointer', async () => {
await sessionBuffer.ingest(
makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })
);
await sessionBuffer.cleanup({
projectId,
deviceId: 'device-Y',
sessionId,
profileId: 'profile-X',
});
expect(await redis.get(`session:${projectId}:device-Y`)).toBeNull();
expect(
await redis.zscore(`session:wallclock:${projectId}`, 'device-Y')
).toBeNull();
expect(
await redis.get(`session:profile:${projectId}:profile-X`)
).toBeNull();
});
it('cleanup() is a no-op when slot is owned by a different session (boundary)', async () => {
// Open session S1
await sessionBuffer.ingest(makePayload({ sessionId: 's1' }));
// 35min later — boundary, opens S2 at same (pid, did)
await sessionBuffer.ingest(
makePayload({
sessionId: 's2',
createdAt: new Date(Date.now() + 35 * 60 * 1000),
})
);
// Try to cleanup S1 — should not touch S2's data. The Lua script in
// cleanup() detects the id mismatch and returns early.
await sessionBuffer.cleanup({
projectId,
deviceId,
sessionId: 's1',
});
const blob = await redis.get(`session:${projectId}:${deviceId}`);
expect(blob).not.toBeNull();
const parsed = JSON.parse(blob ?? '');
expect(parsed.id).toBe('s2');
expect(
await redis.zscore(`session:wallclock:${projectId}`, deviceId)
).not.toBeNull();
});
it('processes buffer and inserts sessions into ClickHouse', async () => {
await sessionBuffer.ingest(makePayload());
await sessionBuffer.processBuffer();
expect(chInsert).toHaveBeenCalledWith(
expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
);
expect(await sessionBuffer.getBufferSize()).toBe(0);
});
it('squash does not orphan the creation row when create + updates share a batch', async () => {
// Regression: a brand-new session whose creation event and subsequent
// updates all land in one flush window. ingest emits (+1,v1) for the
// creation then (-1,vN)/(+1,vN+1) pairs for each update. The squash must
// net per version so nothing is left un-collapsible: a (-1,V) row only
// collapses against a (+1,V) of the SAME version in CH.
const t0 = Date.now();
const EVENTS = 8;
for (let i = 0; i < EVENTS; i++) {
await sessionBuffer.ingest(
makePayload({ createdAt: new Date(t0 + i * 1000) })
);
}
const inserted: Array<{ sign: number; version: number }> = [];
chInsert.mockImplementation(async ({ values }) => {
for (const v of values) {
inserted.push({ sign: v.sign, version: v.version });
}
return await Promise.resolve(undefined);
});
await sessionBuffer.processBuffer();
// Squash must actually fire. The 8 events queue 1 creation + 7
// (-1,vN)/(+1,vN+1) pairs = 15 raw rows; for a fresh session with no
// CH-resident row they all net down to the single final (+1, v8). If
// squash is disabled/removed, every raw row is inserted and this is 15
// — this assertion is the guard that the optimisation is wired in.
expect(inserted.length).toBe(1);
// Net the inserted rows per version exactly as CH's
// VersionedCollapsingMergeTree(sign, version) would: a row survives only
// if positives and negatives at its version don't cancel.
const netByVersion = new Map<number, number>();
for (const { sign, version } of inserted) {
netByVersion.set(version, (netByVersion.get(version) ?? 0) + sign);
}
const survivors = [...netByVersion.entries()].filter(
([, net]) => net !== 0
);
// A fresh session must collapse to exactly one final +1 row, with no
// orphaned negatives (the bug left a permanent (-1, v1)).
expect(survivors).toEqual([[EVENTS, 1]]);
});
it('retains sessions in queue when ClickHouse insert fails', async () => {
await sessionBuffer.ingest(makePayload());
chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
// Errors propagate to tryFlush; the queue is preserved.
await expect(sessionBuffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
expect(await sessionBuffer.getBufferSize()).toBe(1);
});
// The write-back runs once per client event, so every extra command on it is
// multiplied by the whole ingest rate.
it('ingest sends no TYPE, EVAL or EVALSHA to Redis', async () => {
const monitor = await redis.duplicate().monitor();
const commands: string[] = [];
let sentinelSeen = false;
// A dev stack may share this Redis, so only commands on this test's
// project count.
monitor.on('monitor', (_time: string, args: string[]) => {
if (args.includes(MONITOR_SENTINEL)) {
sentinelSeen = true;
} else if (args.some((arg) => arg.includes(projectId))) {
commands.push(String(args[0]).toLowerCase());
}
});
const t0 = Date.now();
await sessionBuffer.ingest(makePayload({ createdAt: new Date(t0) }));
await sessionBuffer.ingest(makePayload({ createdAt: new Date(t0 + 1000) }));
await redis.ping(MONITOR_SENTINEL);
while (!sentinelSeen) {
await Bun.sleep(MONITOR_POLL_MS);
}
monitor.disconnect();
expect(commands).toContain('rpush');
expect(commands).not.toContain('type');
expect(commands).not.toContain('eval');
expect(commands).not.toContain('evalsha');
});
it('ingest never flushes, even past the batch size', async () => {
const buffer = bufferWithBatchSize(2);
const t0 = Date.now();
for (let i = 0; i < 5; i++) {
await buffer.ingest(makePayload({ createdAt: new Date(t0 + i * 1000) }));
}
expect(chInsert).not.toHaveBeenCalled();
expect(await buffer.getBufferSize()).toBe(9);
});
it('a flush drains a backlog larger than one batch', async () => {
const buffer = bufferWithBatchSize(3);
const t0 = Date.now();
for (let i = 0; i < 10; i++) {
await buffer.ingest(
makePayload({ deviceId: `device-${i}`, createdAt: new Date(t0) })
);
}
await buffer.tryFlush({ trigger: 'cron' });
expect(chInsert).toHaveBeenCalledTimes(4);
expect(await buffer.getBufferSize()).toBe(0);
});
it('inserts the chunks of one batch concurrently', async () => {
const config = testCoreConfig();
config.buffers.session.chunkSize = 2;
const buffer = new SessionBuffer(
bufferDepsWithCh({ insert: chInsert }, config)
);
const t0 = Date.now();
for (let i = 0; i < 10; i++) {
await buffer.ingest(
makePayload({
deviceId: `device-${i}`,
sessionId: `session-${i}`,
createdAt: new Date(t0),
})
);
}
let inFlight = 0;
let maxInFlight = 0;
const insertedIds: string[] = [];
chInsert.mockImplementation(async ({ values }) => {
inFlight += 1;
maxInFlight = Math.max(maxInFlight, inFlight);
await Bun.sleep(INSERT_LATENCY_MS);
insertedIds.push(...values.map((row) => row.device_id));
inFlight -= 1;
});
await buffer.processBuffer();
expect(chInsert).toHaveBeenCalledTimes(5);
expect(maxInFlight).toBeGreaterThan(1);
expect(insertedIds.sort()).toEqual(
Array.from({ length: 10 }, (_, i) => `device-${i}`).sort()
);
expect(await buffer.getBufferSize()).toBe(0);
});
it('keeps the whole batch queued when one concurrent chunk fails', async () => {
const config = testCoreConfig();
config.buffers.session.chunkSize = 2;
const buffer = new SessionBuffer(
bufferDepsWithCh({ insert: chInsert }, config)
);
const t0 = Date.now();
for (let i = 0; i < 10; i++) {
await buffer.ingest(
makePayload({
deviceId: `device-${i}`,
sessionId: `session-${i}`,
createdAt: new Date(t0),
})
);
}
chInsert.mockImplementation(async ({ values }) => {
await Bun.sleep(INSERT_LATENCY_MS);
if (values.some((row) => row.device_id === 'device-5')) {
throw new Error('ClickHouse unavailable');
}
});
await expect(buffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
expect(await buffer.getBufferSize()).toBe(10);
});
it('inserts a whole batch in one insert by default', async () => {
const buffer = new SessionBuffer(bufferDepsWithCh({ insert: chInsert }));
// More rows than any fixed chunk size short of the batch size would allow
// in one insert.
const rows = Array.from({ length: ROWS_OVER_OLD_CHUNK_SIZE }, (_, i) =>
JSON.stringify({ id: `session-${i}`, sign: 1, version: 1, duration: 0 })
);
await redis.rpush('session-buffer', ...rows);
await buffer.processBuffer();
expect(chInsert).toHaveBeenCalledTimes(1);
expect(chInsert.mock.calls[0]?.[0].values).toHaveLength(
ROWS_OVER_OLD_CHUNK_SIZE
);
});
it('after a failed chunk starts no new chunk and rejects once the running ones finish', async () => {
const config = testCoreConfig();
config.buffers.session.chunkSize = 2;
const buffer = new SessionBuffer(
bufferDepsWithCh({ insert: chInsert }, config)
);
const t0 = Date.now();
for (let i = 0; i < 20; i++) {
await buffer.ingest(
makePayload({
deviceId: `device-${i}`,
sessionId: `session-${i}`,
createdAt: new Date(t0),
})
);
}
let started = 0;
let finished = 0;
chInsert.mockImplementation(async ({ values }) => {
started += 1;
if (values.some((row) => row.device_id === 'device-0')) {
throw new Error('ClickHouse unavailable');
}
await Bun.sleep(INSERT_LATENCY_MS);
finished += 1;
});
await expect(buffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
// Ten chunks, five at a time: the first wave runs, nothing after it.
expect(started).toBe(BUFFER_CH_INSERT_CONCURRENCY_DEFAULT);
expect(finished).toBe(BUFFER_CH_INSERT_CONCURRENCY_DEFAULT - 1);
expect(await buffer.getBufferSize()).toBe(20);
});
it('reports read, transform, insert and trim timings', async () => {
const buffer = bufferWithBatchSize(1000);
const observations: FlushObservation[] = [];
buffer.flushObserver = (observation) => observations.push(observation);
await buffer.ingest(makePayload());
await buffer.tryFlush({ trigger: 'cron' });
expect(observations).toHaveLength(1);
const [observation] = observations;
expect(observation?.result).toBe('success');
expect(observation?.rowsProcessed).toBe(1);
for (const phase of [
'lrangeMs',
'transformMs',
'chInsertMs',
'trimMs',
] as const) {
expect(observation?.phases?.[phase]).toBeGreaterThanOrEqual(0);
}
});
});
const INSERT_LATENCY_MS = 10;
const ROWS_OVER_OLD_CHUNK_SIZE = 1500;
/** `testCoreConfig()` leaves it unset, so the base buffer's default of 5 applies. */
const BUFFER_CH_INSERT_CONCURRENCY_DEFAULT = 5;
const MONITOR_SENTINEL = 'session-buffer-test-done';
const MONITOR_POLL_MS = 5;
function bufferWithBatchSize(batchSize: number) {
const config = testCoreConfig();
config.buffers.session.batchSize = batchSize;
return new SessionBuffer(bufferDepsWithCh({ insert: chInsert }, config));
}