OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
import { getRedisCache, type Redis } from '@openpanel/redis';
import { DateTime, getSafeJson } from '@openpanel/shared';
import type { CoreConfig } from '../config';
import type { IServiceCreateEventPayload } from '../modules/event/event.service';
import type { IClickhouseSession } from '../modules/session/session.service';
import { TABLE_NAMES } from '../shared/ch-tables';
import { BaseBuffer, type BufferDeps } from './base-buffer';
import { defineRedisScript } from './redis-script';
// 30min of idle in event-time → session ends.
const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;
// The profile pointer is refreshed on every persist, so it only has to outlive
// the idle timeout plus the reaper's lag. It sits in no index, so a TTL is the
// only thing that collects a pointer `cleanup()` never gets handed.
const PROFILE_POINTER_TTL_FACTOR = 2;
/**
* Idle window for a session (boundary detection + the reaper's default
* deadman). Overridable through SESSION_TIMEOUT_MS so the E2E harness can
* shrink it from 30min to a few seconds — the harness and the stack must be
* given the same value (apps/api/e2e/README.md).
*/
export function resolveSessionTimeoutMs(config: CoreConfig): number {
return config.session.timeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
}
/** A session row is rewritten on every event, so its group union is capped. */
const MAX_SESSION_GROUPS = 100;
const sessionKey = (projectId: string, deviceId: string) =>
`session:${projectId}:${deviceId}`;
const profileIndexKey = (projectId: string, profileId: string) =>
`session:profile:${projectId}:${profileId}`;
const wallclockSetKey = (projectId: string) => `session:wallclock:${projectId}`;
const PROJECTS_SET_KEY = 'session:projects';
// Atomic id-gated cleanup. Only deletes the blob + sorted-set entry +
// profile pointer if the live blob's session id matches the one we're
// closing. Prevents races where a boundary has already overwritten the slot
// with a new session.
const cleanupScript = defineRedisScript(`
local cur = redis.call('GET', KEYS[1])
if cur then
local ok, parsed = pcall(cjson.decode, cur)
if ok and parsed and parsed.id ~= ARGV[1] then
return 0
end
end
redis.call('DEL', KEYS[1])
redis.call('ZREM', KEYS[2], ARGV[2])
if KEYS[3] ~= '' then
redis.call('DEL', KEYS[3])
end
return 1
`);
export type SessionIngestResult =
| { kind: 'new'; current: IClickhouseSession }
| { kind: 'extend'; current: IClickhouseSession }
| {
kind: 'boundary';
current: IClickhouseSession;
closed: IClickhouseSession;
};
function toClickhouseDate(date: Date): string {
return DateTime.fromJSDate(date)
.setZone('UTC')
.toFormat('yyyy-MM-dd HH:mm:ss.SSS');
}
function fromClickhouseDate(s: string): Date {
return DateTime.fromFormat(s, 'yyyy-MM-dd HH:mm:ss.SSS', {
zone: 'UTC',
}).toJSDate();
}
function pickUtm(
payload: IServiceCreateEventPayload,
key: 'utm_medium' | 'utm_source' | 'utm_campaign' | 'utm_content' | 'utm_term'
): string {
const query = (
payload.properties as { __query?: Record<string, unknown> } | undefined
)?.__query;
const v = query?.[key];
return v ? String(v) : '';
}
// Large enough that a busy session's versions share a batch and squash.
const DEFAULT_BATCH_SIZE = 10_000;
export class SessionBuffer extends BaseBuffer {
private readonly batchSize =
this.deps.config.buffers.session.batchSize ?? DEFAULT_BATCH_SIZE;
// One insert per batch by default: a failed chunk keeps the whole batch
// queued, so chunks that already landed would be inserted again, and a
// duplicated sign row corrupts the collapsed session.
private readonly chunkSize =
this.deps.config.buffers.session.chunkSize ?? this.batchSize;
private readonly squashEnabled = this.deps.config.buffers.session.squash;
private readonly idleTimeoutMs = resolveSessionTimeoutMs(this.deps.config);
private readonly profilePointerTtlMs =
this.idleTimeoutMs * PROFILE_POINTER_TTL_FACTOR;
private readonly redisKey = 'session-buffer';
private readonly redis: Redis;
constructor(deps: BufferDeps) {
super(deps, {
name: 'session',
onFlush: async () => {
await this.processBuffer();
},
});
this.redis = getRedisCache();
}
/**
* Look up the active session for a device, either directly by deviceId or
* via the profile index (`session:profile:{pid}:{profileId}` → deviceId).
*/
public async getExistingSession(
options:
| { projectId: string; deviceId: string }
| { projectId: string; profileId: string }
): Promise<IClickhouseSession | null> {
if ('deviceId' in options) {
const hit = await this.redis.get(
sessionKey(options.projectId, options.deviceId)
);
return hit ? getSafeJson<IClickhouseSession>(hit) : null;
}
const deviceId = await this.redis.get(
profileIndexKey(options.projectId, options.profileId)
);
if (!deviceId) {
return null;
}
const hit = await this.redis.get(sessionKey(options.projectId, deviceId));
return hit ? getSafeJson<IClickhouseSession>(hit) : null;
}
/** The live sessions of several devices in one MGET, in the given order. */
public async getExistingSessions(
projectId: string,
deviceIds: string[]
): Promise<(IClickhouseSession | null)[]> {
if (deviceIds.length === 0) {
return [];
}
const hits = await this.redis.mget(
deviceIds.map((deviceId) => sessionKey(projectId, deviceId))
);
return hits.map((hit) =>
hit ? getSafeJson<IClickhouseSession>(hit) : null
);
}
/**
* Ingest one event into the session lifecycle. Returns the action taken so
* the caller can drive session_start / session_end event emission —
* `session_start` / `session_end` events themselves are skipped here, since
* they are derived signals, not session activity.
*/
async ingest(
payload: IServiceCreateEventPayload
): Promise<SessionIngestResult | null> {
if (!(payload.projectId && payload.deviceId)) {
return null;
}
if (payload.name === 'session_start' || payload.name === 'session_end') {
return null;
}
try {
const existing = await this.getExistingSession({
projectId: payload.projectId,
deviceId: payload.deviceId,
});
const eventTimeMs = payload.createdAt.getTime();
const isBoundary =
existing &&
eventTimeMs - fromClickhouseDate(existing.ended_at).getTime() >
this.idleTimeoutMs;
if (existing && !isBoundary) {
const { current, chRows } = this.extendSession(existing, payload);
await this.persist(current, chRows, existing);
return { kind: 'extend', current };
}
const current = this.newSession(payload);
await this.persist(current, [current]);
if (existing && isBoundary) {
return { kind: 'boundary', current, closed: existing };
}
return { kind: 'new', current };
} catch (error) {
this.logger.error({ err: error }, 'Failed to ingest session');
return null;
}
}
/**
* Remove a closed session from Redis. Atomically gated on `sessionId` via
* Lua: if a different session now occupies the `(projectId, deviceId)`
* slot (because a boundary already overwrote it with a new session), the
* call is a no-op — the new session still needs its blob and sorted-set
* entry.
*
* Blobs have no Redis TTL, so this is the only path that removes them. The
* daily vacuum cron is a backstop for when it fails (worker crash, network
* partition). The profile pointer additionally expires on its own: it is
* indexed nowhere, so a pointer this call is not handed (a boundary that
* gated the script out, a session re-identified before it closed) would
* otherwise stay forever.
*/
async cleanup({
projectId,
deviceId,
sessionId,
profileId,
}: {
projectId: string;
deviceId: string;
sessionId: string;
profileId?: string | null;
}): Promise<void> {
const pointerKey =
profileId && profileId !== deviceId
? profileIndexKey(projectId, profileId)
: '';
await cleanupScript(
this.redis,
[sessionKey(projectId, deviceId), wallclockSetKey(projectId), pointerKey],
[sessionId, deviceId]
);
}
/**
* Extend an in-flight session with a new event. Returns the updated
* session AND the pair of CollapsingMergeTree rows that ClickHouse needs
* to replace the previous row (sign=-1 cancels, sign=+1 inserts).
*/
private extendSession(
existing: IClickhouseSession,
payload: IServiceCreateEventPayload
): {
current: IClickhouseSession;
chRows: [IClickhouseSession, IClickhouseSession];
} {
const oldRow: IClickhouseSession = { ...existing, sign: -1 };
const current: IClickhouseSession = {
...existing,
sign: 1,
version: existing.version + 1,
};
// Out-of-order safety: treat the session window as [min(event ts),
// max(event ts)]. Late-arriving events (offline mobile flush, retries)
// can't drag ended_at backwards or push duration negative.
const eventTimeMs = payload.createdAt.getTime();
const startMs = fromClickhouseDate(current.created_at).getTime();
const endMs = fromClickhouseDate(current.ended_at).getTime();
const eventCh = toClickhouseDate(payload.createdAt);
if (eventTimeMs >= endMs) {
current.ended_at = eventCh;
if (payload.path) {
current.exit_path = payload.path;
}
if (payload.origin) {
current.exit_origin = payload.origin;
}
}
if (eventTimeMs < startMs) {
current.created_at = eventCh;
if (payload.path) {
current.entry_path = payload.path;
}
if (payload.origin) {
current.entry_origin = payload.origin;
}
} else {
if (!current.entry_path && payload.path) {
current.entry_path = payload.path;
}
if (!current.entry_origin && payload.origin) {
current.entry_origin = payload.origin;
}
}
current.duration =
fromClickhouseDate(current.ended_at).getTime() -
fromClickhouseDate(current.created_at).getTime();
if (payload.name === 'revenue') {
current.revenue = (current.revenue ?? 0) + (payload.revenue ?? 0);
}
if (payload.name === 'screen_view' && payload.path) {
current.screen_view_count += 1;
} else {
current.event_count += 1;
}
if (current.screen_view_count > 1) {
current.is_bounce = false;
}
if (payload.profileId && payload.profileId !== payload.deviceId) {
current.profile_id = payload.profileId;
}
if (payload.groups?.length) {
current.groups = [
...new Set([...(current.groups ?? []), ...payload.groups]),
].slice(0, MAX_SESSION_GROUPS);
}
return { current, chRows: [current, oldRow] };
}
/**
* Open a brand-new session from this event's payload. Called for the first
* event of a device AND on boundary (gap > 30min closing the old session).
*/
private newSession(payload: IServiceCreateEventPayload): IClickhouseSession {
const createdAt = toClickhouseDate(payload.createdAt);
// For anonymous traffic the SDK doesn't send a profileId — fall back to
// the deviceId so each anonymous device shows up as a unique visitor in
// the dashboard. Matches what `createEvent` does for the events table.
const profileId = payload.profileId || payload.deviceId;
return {
id: payload.sessionId,
project_id: payload.projectId,
device_id: payload.deviceId,
profile_id: profileId,
is_bounce: true,
created_at: createdAt,
ended_at: createdAt,
event_count: payload.name === 'screen_view' ? 0 : 1,
screen_view_count: payload.name === 'screen_view' ? 1 : 0,
entry_path: payload.path ?? '',
entry_origin: payload.origin ?? '',
exit_path: payload.path ?? '',
exit_origin: payload.origin ?? '',
revenue: payload.name === 'revenue' ? (payload.revenue ?? 0) : 0,
referrer: payload.referrer ?? '',
referrer_name: payload.referrerName ?? '',
referrer_type: payload.referrerType ?? '',
os: payload.os ?? '',
os_version: payload.osVersion ?? '',
browser: payload.browser ?? '',
browser_version: payload.browserVersion ?? '',
device: payload.device ?? '',
brand: payload.brand ?? '',
model: payload.model ?? '',
country: payload.country ?? '',
region: payload.region ?? '',
city: payload.city ?? '',
longitude: payload.longitude ?? null,
latitude: payload.latitude ?? null,
duration: payload.duration ?? 0,
utm_medium: pickUtm(payload, 'utm_medium'),
utm_source: pickUtm(payload, 'utm_source'),
utm_campaign: pickUtm(payload, 'utm_campaign'),
utm_content: pickUtm(payload, 'utm_content'),
utm_term: pickUtm(payload, 'utm_term'),
groups: (payload.groups ?? []).slice(0, MAX_SESSION_GROUPS),
sign: 1,
version: 1,
};
}
/**
* Write-back in one MULTI: session blob + wallclock ZSET + projects SET +
* profile index + ClickHouse buffer rows. Append only; the cron drains the
* buffer, so ingest never waits on a ClickHouse insert.
*
* No TTL on the blob — it is removed exclusively by `cleanup()` after
* `session_end` emission, so the reaper can always find a session's state
* regardless of how long the worker has been down. The reaper never goes
* through the profile pointer, so that one may expire; the only reader is
* server-side ingest looking for a live client session to ride along on.
*/
private async persist(
current: IClickhouseSession,
chRows: IClickhouseSession[],
previous?: IClickhouseSession
) {
const projectId = current.project_id;
const deviceId = current.device_id;
const hasProfilePointer =
current.profile_id && current.profile_id !== current.device_id;
const previousProfileId = previous?.profile_id;
const hasStalePointer =
previousProfileId &&
previousProfileId !== deviceId &&
previousProfileId !== current.profile_id;
const multi = this.redis
.multi()
.set(sessionKey(projectId, deviceId), JSON.stringify(current))
.zadd(wallclockSetKey(projectId), Date.now(), deviceId)
.sadd(PROJECTS_SET_KEY, projectId);
if (hasStalePointer) {
multi.del(profileIndexKey(projectId, previousProfileId));
}
if (hasProfilePointer) {
multi.set(
profileIndexKey(projectId, current.profile_id),
deviceId,
'PX',
this.profilePointerTtlMs
);
}
multi.rpush(this.redisKey, ...chRows.map((row) => JSON.stringify(row)));
// Ioredis resolves a MULTI whose commands failed, with the error per entry.
const results = await multi.exec();
if (results === null) {
throw new Error('session write-back transaction was aborted');
}
const rejected = results.find(([commandError]) => commandError !== null);
if (rejected?.[0]) {
throw rejected[0];
}
}
/**
* Squash the (sign=-1, sign=+1) rows for each session id down to the
* minimal set that produces the same collapsed state in ClickHouse, by
* netting the signs per version. Single-row sessions pass through
* unchanged.
*
* `sessions` uses VersionedCollapsingMergeTree(sign, version): a (-1, V)
* row only collapses against a (+1, V) row of the SAME version. So the
* net effect of a batch on a session is computed per-version — sum the
* signs at each version and emit only the rows needed to realise that
* net. Versions that fully cancel (net 0) are dropped; the surviving
* rows are normally the final (+1, maxVersion) plus, for a mid-life
* session, the (-1, V) that cancels CH's resident row.
*
* This per-version netting is required because a session's creation row
* (+1 at the base version) can land in the SAME batch as its updates.
* The first update emits a (-1, baseVersion) whose only partner is that
* in-batch creation (+1, baseVersion). They must cancel each other here;
* naively keeping "oldest -1 + newest +1" would drop the creation +1
* while keeping its canceller, leaving a permanently un-collapsible (-1)
* row in CH (duplicate session rows + negative bounce counts).
*/
private squashSessionsByVersion(
sessions: IClickhouseSession[]
): IClickhouseSession[] {
if (sessions.length <= 1) {
return sessions;
}
const grouped = new Map<string, IClickhouseSession[]>();
for (const s of sessions) {
const arr = grouped.get(s.id);
if (arr) {
arr.push(s);
} else {
grouped.set(s.id, [s]);
}
}
let squashedCount = 0;
const out: IClickhouseSession[] = [];
for (const entries of grouped.values()) {
if (entries.length === 1) {
out.push(entries[0]!);
continue;
}
const netByVersion = new Map<
number,
{ net: number; pos?: IClickhouseSession; neg?: IClickhouseSession }
>();
for (const e of entries) {
const slot = netByVersion.get(e.version) ?? { net: 0 };
slot.net += e.sign;
if (e.sign === 1) {
slot.pos = e;
} else if (e.sign === -1) {
slot.neg = e;
}
netByVersion.set(e.version, slot);
}
const emitted: IClickhouseSession[] = [];
for (const slot of netByVersion.values()) {
if (slot.net === 0) {
continue;
}
const row = slot.net > 0 ? slot.pos : slot.neg;
if (!row) {
continue;
}
for (let i = 0; i < Math.abs(slot.net); i++) {
emitted.push(row);
}
}
squashedCount += entries.length - emitted.length;
out.push(...emitted);
}
if (squashedCount > 0) {
this.logger.debug(
{
inputRows: sessions.length,
outputRows: out.length,
dropped: squashedCount,
uniqueSessions: grouped.size,
},
'Session batch squashed'
);
}
return out;
}
async processBuffer() {
await this.drainBatches(this.batchSize, () => this.processBatch());
}
/** Inserts and trims one batch from the head of the list; returns its size. */
private async processBatch(): Promise<number> {
const lrangeStart = performance.now();
const events = await this.redis.lrange(
this.redisKey,
0,
this.batchSize - 1
);
const lrangeMs = performance.now() - lrangeStart;
if (events.length === 0) {
this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });
return 0;
}
const transformStart = performance.now();
const parsed: IClickhouseSession[] = [];
for (const e of events) {
const s = getSafeJson<IClickhouseSession>(e);
if (!s) {
continue;
}
// Freshly parsed object — safe to clamp in place, no clone needed.
s.duration = Math.max(0, s.duration || 0);
parsed.push(s);
}
// A busy session appends a (-1, +1) pair per extend within one flush.
// VersionedCollapsingMergeTree would collapse them at merge time anyway,
// but inserting them all costs network, gzip CPU and merge work. Squash
// per version to insert only the boundary rows.
//
// Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.
const sessions = this.squashEnabled
? this.squashSessionsByVersion(parsed)
: parsed;
const transformMs = performance.now() - transformStart;
// Chunks may land in any order: VersionedCollapsingMergeTree collapses by
// (sign, version), not by insert order.
const ch = this.resolveCh();
const chStart = performance.now();
await this.parallelLimit(this.chunks(sessions, this.chunkSize), (chunk) =>
ch.insert({
table: TABLE_NAMES.sessions,
values: chunk,
format: 'JSONEachRow',
})
);
const chInsertMs = performance.now() - chStart;
// Trim only after a successful insert — on failure the rows stay queued.
// Don't swallow: let it propagate so the base flush records result:'error'
// (and the next flush retries the still-queued rows).
const trimStart = performance.now();
await this.redis.ltrim(this.redisKey, events.length, -1);
const trimMs = performance.now() - trimStart;
this.reportFlushStats({
rowsProcessed: events.length,
phases: { lrangeMs, transformMs, chInsertMs, trimMs },
});
return events.length;
}
protected getRedisListKey(): string {
return this.redisKey;
}
}