OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat: new session management (#393)
Carl-Gerhard Lindesvärdbc260da3812a
diff --git a/apps/api/e2e/README.md b/apps/api/e2e/README.mdnew file mode 100644index 000000000..8487466e1--- /dev/null+++ b/apps/api/e2e/README.md@@ -0,0 +1,54 @@+# Session E2E++Two harnesses over a shared foundation (`lib.ts`), both driving the **real stack**+over HTTP and asserting state in **both ClickHouse and Redis**:++- `session-e2e.ts` (`e2e:sessions`) — **correctness**: the full lifecycle for one+ session per scenario (open/extend/close via reaper + boundary, replay, identify),+ including Redis cleanup.+- `session-stress.ts` (`e2e:sessions:stress`) — **volume + drain**: ramps out many+ sessions, then drives reaper + buffer flushes until *everything* has drained+ (every `session_end` emitted, Redis cleaned, session buffer empty) and reconciles+ the ClickHouse counts. Exits only when nothing is left open.++## What it covers++| Scenario | Asserts |+|----------|---------|+| Single session → reaper close | session blob + wallclock + projects-set in Redis; one `session_start` + N events in CH; after the reaper closes it: one `session_end`, a collapsed `sessions` row, and Redis fully cleaned (blob + wallclock gone, idempotency claim present). |+| Boundary split | a >idle-window gap opens a NEW session id and emits a `session_end` for the first + a `session_start` for the second. |+| Replay | a replay chunk lands in `session_replay_chunks` under the echoed session id. |+| Identify | `session:profile:{pid}:{profileId}` pointer written; events carry the identified `profile_id`. |++## Running++Sessions idle out after 30 min by default. Shrink that and start the stack with+the **same** value the harness uses, then run the harness:++```bash+# 1. Start the stack with a short idle window (Docker must be up: pnpm dock:up)+SESSION_TIMEOUT_MS=4000 pnpm dev++# 2. In another terminal, run the harness with the SAME timeout+SESSION_TIMEOUT_MS=4000 pnpm --filter @openpanel/api e2e:sessions++# …or the stress + drain test (500 sessions by default)+SESSION_TIMEOUT_MS=4000 pnpm --filter @openpanel/api e2e:sessions:stress+```++Stress tunables (env): `E2E_SESSIONS` (500), `E2E_CONCURRENCY` (25),+`E2E_EVENTS_PER_SESSION` (3), `E2E_DRAIN_TIMEOUT_MS` (120000).++It exits non-zero if any check fails and prints a summary. Total run is ~30–60s+with a 4s window (each close waits roughly one idle window).++The harness triggers the reaper on demand via the worker's `/debug/cron`+endpoint, so it never waits for the 5-minute reaper cron.++### Notes+- Uses a dedicated, isolated project (`e2e-sessions`) and a throwaway client+ (`ignoreCorsAndSecret`), created/upserted automatically under org `openpanel-dev`.+- Each run uses fresh device IPs, so reruns don't collide with prior state.+- Overridable: `E2E_API_URL` (default `:3333`), `E2E_WORKER_URL` (default `:9999`).+- The harness and the stack **must share the same `SESSION_TIMEOUT_MS`** — the+ harness derives its idle waits from it.diff --git a/apps/api/e2e/lib.ts b/apps/api/e2e/lib.tsnew file mode 100644index 000000000..42378e5a0--- /dev/null+++ b/apps/api/e2e/lib.ts@@ -0,0 +1,231 @@+/**+ * Shared building blocks for the session E2E + stress harnesses:+ * config, an HTTP track client, the reaper trigger, Redis/ClickHouse helpers,+ * fixtures, polling, and a tiny check/report framework.+ */++import { ClientType, chQuery, db, getClientByIdCached } from '@openpanel/db';+import { getRedisCache } from '@openpanel/redis';++// ── Config ──────────────────────────────────────────────────────────────────+export const API_URL = process.env.E2E_API_URL || 'http://localhost:3333';+export const WORKER_URL = process.env.E2E_WORKER_URL || 'http://localhost:9999';+export const ORG_ID = 'openpanel-dev';+export const PROJECT_ID = 'e2e-sessions';+export const CLIENT_ID = 'e2e1e2e1-0000-4000-8000-000000000001';+export const UA =+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';++export const SESSION_TIMEOUT_MS = Number.parseInt(+ process.env.SESSION_TIMEOUT_MS || String(1000 * 60 * 30),+ 10+);+/** How long to wait for a session to fall outside its idle window before closing. */+export const IDLE_WAIT_MS = SESSION_TIMEOUT_MS + 2000;++export const redis = getRedisCache();+export const runId = Date.now();++export { chQuery };++// ── Redis key helpers ───────────────────────────────────────────────────────+export const sessionKey = (deviceId: string) => `session:${PROJECT_ID}:${deviceId}`;+export const wallclockKey = `session:wallclock:${PROJECT_ID}`;+export const profileKey = (profileId: string) =>+ `session:profile:${PROJECT_ID}:${profileId}`;+export const claimKey = (deviceId: string, sessionId: string) =>+ `session:end:emitted:${PROJECT_ID}:${deviceId}:${sessionId}`;+/** The session-buffer's Redis list (ground-truth pending CH rows). */+export const SESSION_BUFFER_LIST = 'session-buffer';++export async function getBlob(deviceId: string) {+ const raw = await redis.get(sessionKey(deviceId));+ return raw ? JSON.parse(raw) : null;+}++// ── Timing ──────────────────────────────────────────────────────────────────+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));++/** Poll `fn` until it returns a truthy value or the timeout elapses. */+export async function pollUntil<T>(+ fn: () => Promise<T>,+ { timeoutMs = 30_000, intervalMs = 750 } = {}+): Promise<T | null> {+ const deadline = Date.now() + timeoutMs;+ // biome-ignore lint/nursery/noConstantCondition: poll loop+ while (true) {+ const value = await fn();+ if (value) return value;+ if (Date.now() >= deadline) return null;+ await sleep(intervalMs);+ }+}++// ── Report framework ──────────────────────────────────────────────────────+type Result = { scenario: string; name: string; ok: boolean; detail?: string };+const results: Result[] = [];+let currentScenario = 'setup';++export function scenario(name: string) {+ currentScenario = name;+ console.log(`\n▶ ${name}`);+}+export function check(name: string, ok: boolean, detail?: string) {+ results.push({ scenario: currentScenario, name, ok, detail });+ console.log(` ${ok ? '✓' : '✗'} ${name}${detail && !ok ? ` — ${detail}` : ''}`);+}+/** Print the summary and return the number of failed checks. */+export function summarize(): number {+ const failed = results.filter((r) => !r.ok);+ console.log(`\n${'─'.repeat(60)}`);+ console.log(`${results.length - failed.length}/${results.length} checks passed`);+ if (failed.length) {+ console.log('\nFailures:');+ for (const f of failed) {+ console.log(` ✗ [${f.scenario}] ${f.name}${f.detail ? ` — ${f.detail}` : ''}`);+ }+ }+ return failed.length;+}++// ── HTTP ────────────────────────────────────────────────────────────────────+export type TrackResponse = { deviceId: string; sessionId: string };++export async function track(body: unknown, ip: string): Promise<TrackResponse> {+ const res = await fetch(`${API_URL}/track`, {+ method: 'POST',+ headers: {+ 'content-type': 'application/json',+ 'openpanel-client-id': CLIENT_ID,+ 'user-agent': UA,+ 'x-client-ip': ip,+ },+ body: JSON.stringify(body),+ });+ if (!res.ok) {+ throw new Error(`POST /track ${res.status}: ${await res.text()}`);+ }+ return (await res.json()) as TrackResponse;+}++export const screenView = (+ ip: string,+ path: string,+ extra: Record<string, unknown> = {}+) =>+ track(+ {+ type: 'track',+ payload: {+ name: 'screen_view',+ properties: { __path: `https://e2e.test${path}`, __ip: ip, ...extra },+ },+ },+ ip+ );++/** Run a worker cron on demand via the local /debug/cron endpoint. */+export async function triggerCron(type: string) {+ const res = await fetch(`${WORKER_URL}/debug/cron/${type}`, {+ method: 'POST',+ headers: { accept: 'application/json' },+ });+ if (!res.ok) {+ throw new Error(`trigger cron ${type} ${res.status}: ${await res.text()}`);+ }+}++export const triggerReaper = () => triggerCron('sessionReaper');++// ── ClickHouse counting (scoped to a set of session ids for run isolation) ──+const quoteList = (ids: string[]) =>+ ids.map((id) => `'${id.replace(/'/g, "")}'`).join(',');++export async function countByName(+ sessionIds: string[],+ name: string+): Promise<number> {+ if (sessionIds.length === 0) return 0;+ const rows = await chQuery<{ c: string }>(+ `SELECT count() AS c FROM events WHERE project_id = '${PROJECT_ID}' AND name = '${name}' AND session_id IN (${quoteList(sessionIds)})`+ );+ return Number(rows[0]?.c ?? 0);+}++// ── Fixtures ────────────────────────────────────────────────────────────────+export async function ensureFixtures() {+ scenario('setup: project + client');+ try {+ await db.organization.upsert({+ where: { id: ORG_ID },+ create: { id: ORG_ID, name: 'OpenPanel Dev' },+ update: {},+ });+ await db.project.upsert({+ where: { id: PROJECT_ID },+ create: { id: PROJECT_ID, name: 'E2E Sessions', organizationId: ORG_ID },+ update: {},+ });+ await db.client.upsert({+ where: { id: CLIENT_ID },+ create: {+ id: CLIENT_ID,+ name: 'e2e',+ organizationId: ORG_ID,+ projectId: PROJECT_ID,+ type: ClientType.write,+ ignoreCorsAndSecret: true,+ secret: null,+ },+ update: { ignoreCorsAndSecret: true, projectId: PROJECT_ID },+ });+ await getClientByIdCached.clear(CLIENT_ID);+ check('fixtures ready', true);+ } catch (error) {+ check('fixtures ready', false, (error as Error).message);+ throw error;+ }+}++export async function preflight() {+ scenario('preflight');+ const api = await fetch(`${API_URL}/`)+ .then((r) => r.ok || r.status === 404)+ .catch(() => false);+ check(`api reachable at ${API_URL}`, !!api);+ const worker = await fetch(`${WORKER_URL}/debug/cron`)+ .then((r) => r.ok)+ .catch(() => false);+ check(`worker debug reachable at ${WORKER_URL}`, !!worker);+ if (!api || !worker) {+ throw new Error('Stack not reachable — start it with `pnpm dev` first.');+ }+ if (SESSION_TIMEOUT_MS > 60_000) {+ console.warn(+ `\n⚠ SESSION_TIMEOUT_MS=${SESSION_TIMEOUT_MS}ms — this run will be slow.\n` ++ ' Re-run the stack AND the harness with e.g. SESSION_TIMEOUT_MS=4000.'+ );+ }+}++/** Run `fn` over `items` with at most `concurrency` in flight. */+export async function runPool<T>(+ items: T[],+ concurrency: number,+ fn: (item: T, index: number) => Promise<void>+): Promise<void> {+ let next = 0;+ const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {+ while (true) {+ const i = next++;+ if (i >= items.length) return;+ await fn(items[i]!, i);+ }+ });+ await Promise.all(workers);+}++export async function shutdown(failed: number): Promise<never> {+ await redis.quit().catch(() => {});+ process.exit(failed ? 1 : 0);+}diff --git a/apps/api/e2e/session-e2e.ts b/apps/api/e2e/session-e2e.tsnew file mode 100644index 000000000..204d8a40a--- /dev/null+++ b/apps/api/e2e/session-e2e.ts@@ -0,0 +1,268 @@+/**+ * Session correctness E2E.+ *+ * Drives the REAL stack over HTTP and asserts the resulting state in BOTH+ * ClickHouse and Redis through the full lifecycle — open, extend, close (via+ * reaper AND via boundary), replay, identify — including Redis cleanup.+ *+ * Run (shrink the idle window; start the stack with the SAME value):+ * SESSION_TIMEOUT_MS=4000 pnpm dev+ * SESSION_TIMEOUT_MS=4000 pnpm --filter @openpanel/api e2e:sessions+ *+ * For a high-volume drain test see session-stress.ts (`e2e:sessions:stress`).+ */++import {+ chQuery,+ check,+ countByName,+ ensureFixtures,+ getBlob,+ IDLE_WAIT_MS,+ PROJECT_ID,+ pollUntil,+ preflight,+ redis,+ runId,+ scenario,+ screenView,+ SESSION_TIMEOUT_MS,+ shutdown,+ sleep,+ summarize,+ track,+ triggerReaper,+ wallclockKey,+ claimKey,+ sessionKey,+ profileKey,+ WORKER_URL,+ API_URL,+} from './lib';++async function scenarioSingleSession() {+ scenario('single session → reaper close → cleanup');+ const ip = `10.${(runId >> 16) & 255}.1.${runId & 255 || 1}`;++ const first = await screenView(ip, '/a');+ await screenView(ip, '/b');+ await screenView(ip, '/c');+ const { deviceId, sessionId } = first;+ check('track returned a sessionId', !!sessionId, `got '${sessionId}'`);++ const blob = await pollUntil(async () => {+ const b = await getBlob(deviceId);+ return b?.id === sessionId ? b : null;+ });+ check('redis: session blob exists with matching id', !!blob);+ check(+ 'redis: device in wallclock index',+ (await redis.zscore(wallclockKey, deviceId)) !== null+ );+ check(+ 'redis: project in active set',+ (await redis.sismember('session:projects', PROJECT_ID)) === 1+ );++ const starts = await pollUntil(async () => {+ const c = await countByName([sessionId], 'session_start');+ return c > 0 ? c : null;+ });+ check('clickhouse: exactly one session_start', starts === 1, `got ${starts}`);+ const views = await pollUntil(async () => {+ const c = await countByName([sessionId], 'screen_view');+ return c >= 3 ? c : null;+ });+ check('clickhouse: 3 screen_views ingested', !!views, `got ${views}`);++ console.log(` …waiting ${IDLE_WAIT_MS}ms for idle window, then reaping`);+ await sleep(IDLE_WAIT_MS);+ await triggerReaper();++ const ends = await pollUntil(async () => {+ const c = await countByName([sessionId], 'session_end');+ return c > 0 ? c : null;+ });+ check('clickhouse: exactly one session_end', ends === 1, `got ${ends}`);++ const sessionRow = await pollUntil(async () => {+ const rows = await chQuery<{ is_bounce: boolean }>(+ `SELECT is_bounce FROM sessions FINAL WHERE project_id = '${PROJECT_ID}' AND id = '${sessionId}' LIMIT 1`+ );+ return rows[0] ?? null;+ });+ check('clickhouse: sessions row present (collapsed)', !!sessionRow);+ check(+ 'clickhouse: session not a bounce (3 screen_views)',+ sessionRow?.is_bounce === false || Number(sessionRow?.is_bounce) === 0,+ `is_bounce=${sessionRow?.is_bounce}`+ );++ const cleaned = await pollUntil(async () =>+ (await redis.get(sessionKey(deviceId))) === null ? true : null+ );+ check('redis: session blob removed after close', !!cleaned);+ check(+ 'redis: wallclock entry removed after close',+ (await redis.zscore(wallclockKey, deviceId)) === null+ );+ check(+ 'redis: session_end idempotency claim recorded',+ (await redis.get(claimKey(deviceId, sessionId))) !== null+ );+}++async function scenarioBoundary() {+ scenario('boundary split → first session closes, second opens');+ const ip = `10.${(runId >> 16) & 255}.2.${runId & 255 || 1}`;++ const a = await screenView(ip, '/x');+ console.log(` …waiting ${IDLE_WAIT_MS}ms so the next event crosses the boundary`);+ await sleep(IDLE_WAIT_MS);+ const b = await screenView(ip, '/y');++ check(+ 'second event opened a NEW session id',+ a.sessionId !== b.sessionId && !!b.sessionId,+ `${a.sessionId} vs ${b.sessionId}`+ );++ const startsA = await pollUntil(async () =>+ (await countByName([a.sessionId], 'session_start')) > 0 ? true : null+ );+ check('clickhouse: session_start for first session', !!startsA);+ const startsB = await pollUntil(async () =>+ (await countByName([b.sessionId], 'session_start')) > 0 ? true : null+ );+ check('clickhouse: session_start for second session', !!startsB);+ const endsA = await pollUntil(async () =>+ (await countByName([a.sessionId], 'session_end')) > 0 ? true : null+ );+ check('clickhouse: first session got a session_end', !!endsA);+}++async function scenarioNonScreenViewFirst() {+ scenario('session opened by a non-screen_view event');+ const ip = `10.${(runId >> 16) & 255}.5.${runId & 255 || 1}`;++ // First (and only) event is a custom event with no __path.+ const { deviceId, sessionId } = await track(+ { type: 'track', payload: { name: 'purchase', properties: { __ip: ip } } },+ ip+ );+ check('track returned a sessionId for a non-screen_view first event', !!sessionId);++ const starts = await pollUntil(async () =>+ (await countByName([sessionId], 'session_start')) > 0 ? true : null+ );+ check('clickhouse: session_start emitted for a custom-event session', !!starts);++ console.log(` …waiting ${IDLE_WAIT_MS}ms for idle window, then reaping`);+ await sleep(IDLE_WAIT_MS);+ await triggerReaper();++ const ends = await pollUntil(async () =>+ (await countByName([sessionId], 'session_end')) > 0 ? true : null+ );+ check('clickhouse: session_end emitted (closes normally)', !!ends);++ const row = await pollUntil(async () => {+ const rows = await chQuery<{+ screen_view_count: number;+ event_count: number;+ is_bounce: boolean;+ }>(+ `SELECT screen_view_count, event_count, is_bounce FROM sessions FINAL WHERE project_id = '${PROJECT_ID}' AND id = '${sessionId}' LIMIT 1`+ );+ return rows[0] ?? null;+ });+ check('clickhouse: screen_view_count is 0', Number(row?.screen_view_count) === 0, `${row?.screen_view_count}`);+ check('clickhouse: event_count is 1', Number(row?.event_count) === 1, `${row?.event_count}`);+ check(+ 'clickhouse: is_bounce is true (no pageviews)',+ row?.is_bounce === true || Number(row?.is_bounce) === 1,+ `is_bounce=${row?.is_bounce}`+ );++ const cleaned = await pollUntil(async () =>+ (await redis.get(sessionKey(deviceId))) === null ? true : null+ );+ check('redis: session cleaned up after close', !!cleaned);+}++async function scenarioReplay() {+ scenario('replay chunk files under the echoed session id');+ const ip = `10.${(runId >> 16) & 255}.3.${runId & 255 || 1}`;++ const { sessionId } = await screenView(ip, '/replay');+ await track(+ {+ type: 'replay',+ payload: {+ sessionId,+ chunk_index: 0,+ events_count: 1,+ is_full_snapshot: true,+ started_at: new Date(runId).toISOString(),+ ended_at: new Date(runId + 1000).toISOString(),+ payload: '[]',+ },+ },+ ip+ );++ const found = await pollUntil(async () => {+ const rows = await chQuery<{ c: string }>(+ `SELECT count() AS c FROM session_replay_chunks WHERE project_id = '${PROJECT_ID}' AND session_id = '${sessionId}'`+ );+ return Number(rows[0]?.c ?? 0) > 0 ? true : null;+ });+ check('clickhouse: replay chunk stored under the session id', !!found);+}++async function scenarioIdentify() {+ scenario('identified visit → profile index + profile_id on events');+ const ip = `10.${(runId >> 16) & 255}.4.${runId & 255 || 1}`;+ const profileId = `e2e-user-${runId}`;++ const { deviceId, sessionId } = await screenView(ip, '/account');+ await track(+ {+ type: 'track',+ payload: { name: 'signed_in', profileId, properties: { __ip: ip } },+ },+ ip+ );++ const ptr = await pollUntil(async () =>+ (await redis.get(profileKey(profileId))) === deviceId ? true : null+ );+ check('redis: profile→device pointer written', !!ptr);++ const profiled = await pollUntil(async () => {+ const rows = await chQuery<{ c: string }>(+ `SELECT count() AS c FROM events WHERE project_id = '${PROJECT_ID}' AND session_id = '${sessionId}' AND profile_id = '${profileId}'`+ );+ return Number(rows[0]?.c ?? 0) > 0 ? true : null;+ });+ check('clickhouse: event carries the identified profile_id', !!profiled);+}++async function main() {+ console.log(+ `Session E2E — api=${API_URL} worker=${WORKER_URL} timeout=${SESSION_TIMEOUT_MS}ms project=${PROJECT_ID}`+ );+ await preflight();+ await ensureFixtures();+ await scenarioSingleSession();+ await scenarioBoundary();+ await scenarioNonScreenViewFirst();+ await scenarioReplay();+ await scenarioIdentify();+ await shutdown(summarize());+}++main().catch(async (error) => {+ console.error('\nFATAL:', error);+ await shutdown(1);+});diff --git a/apps/api/e2e/session-stress.ts b/apps/api/e2e/session-stress.tsnew file mode 100644index 000000000..907a90d20--- /dev/null+++ b/apps/api/e2e/session-stress.ts@@ -0,0 +1,235 @@+/**+ * Session stress + drain-to-completion.+ *+ * Ramps out many short sessions (default 500, not all at once), then drives the+ * reaper + buffer flushes until EVERYTHING has fully drained — every session_end+ * emitted, Redis cleaned, the session buffer empty — and reconciles the+ * ClickHouse counts. Exits 0 only when nothing is left open.+ *+ * Run (shrink the idle window; start the stack with the SAME value):+ * SESSION_TIMEOUT_MS=4000 pnpm dev+ * SESSION_TIMEOUT_MS=4000 pnpm --filter @openpanel/api e2e:sessions:stress+ *+ * Tunables (env): E2E_SESSIONS (500), E2E_CONCURRENCY (25),+ * E2E_EVENTS_PER_SESSION (3), E2E_DRAIN_TIMEOUT_MS (120000).+ */++import {+ API_URL,+ check,+ chQuery,+ countByName,+ ensureFixtures,+ getBlob,+ IDLE_WAIT_MS,+ pollUntil,+ preflight,+ PROJECT_ID,+ redis,+ runId,+ runPool,+ scenario,+ screenView,+ SESSION_BUFFER_LIST,+ SESSION_TIMEOUT_MS,+ shutdown,+ sleep,+ summarize,+ track,+ triggerCron,+ triggerReaper,+ wallclockKey,+ WORKER_URL,+} from './lib';++const SESSIONS = Number.parseInt(process.env.E2E_SESSIONS || '500', 10);+const CONCURRENCY = Number.parseInt(process.env.E2E_CONCURRENCY || '25', 10);+const EVENTS_PER_SESSION = Number.parseInt(+ process.env.E2E_EVENTS_PER_SESSION || '3',+ 10+);+const DRAIN_TIMEOUT_MS = Number.parseInt(+ process.env.E2E_DRAIN_TIMEOUT_MS || '120000',+ 10+);++type Session = { sessionId: string; deviceId: string };++// Unique IP per session → unique device → unique session. Namespaced by runId+// so reruns and the correctness harness (10.x) never collide.+const ipForSession = (i: number) =>+ `100.${(runId >> 8) & 255}.${(i >> 8) & 255}.${i & 255}`;++// A realistic-ish mix: events are NOT all screen_views.+const CUSTOM_EVENTS = [+ 'sign_up',+ 'login',+ 'add_to_cart',+ 'purchase',+ 'search',+ 'button_click',+ 'video_play',+ 'share',+];+const WORDS = ['alpha', 'beta', 'gamma', 'sverige', 'katt', 'planet'];+const pick = <T>(arr: T[]): T => arr[Math.floor(Math.random() * arr.length)]!;++function randomProps(ip: string): Record<string, unknown> {+ const props: Record<string, unknown> = { __ip: ip };+ const n = Math.floor(Math.random() * 4); // 0–3 extra properties+ for (let k = 0; k < n; k++) {+ props[`prop_${pick(WORDS)}`] =+ Math.random() < 0.5 ? pick(WORDS) : Math.floor(Math.random() * 1000);+ }+ return props;+}++// One event in a session: the entry is always a screen_view; the rest are a+// 50/50 mix of more screen_views and random custom events.+function sendSessionEvent(ip: string, i: number, e: number) {+ if (e === 0 || Math.random() < 0.5) {+ return screenView(ip, `/p${i}/${e}`, randomProps(ip));+ }+ return track(+ { type: 'track', payload: { name: pick(CUSTOM_EVENTS), properties: randomProps(ip) } },+ ip+ );+}++async function emit(): Promise<Session[]> {+ scenario(`emit ${SESSIONS} sessions × ${EVENTS_PER_SESSION} events (concurrency ${CONCURRENCY})`);+ const sessions: Session[] = [];+ let done = 0;+ let errors = 0;+ const started = Date.now();++ await runPool(+ Array.from({ length: SESSIONS }, (_, i) => i),+ CONCURRENCY,+ async (i) => {+ const ip = ipForSession(i);+ try {+ const first = await sendSessionEvent(ip, i, 0); // entry is a screen_view+ for (let e = 1; e < EVENTS_PER_SESSION; e++) {+ await sendSessionEvent(ip, i, e);+ }+ sessions.push({ sessionId: first.sessionId, deviceId: first.deviceId });+ } catch (error) {+ errors++;+ if (errors <= 3) console.warn(` session ${i} failed: ${(error as Error).message}`);+ }+ done++;+ if (done % 100 === 0 || done === SESSIONS) {+ console.log(` …emitted ${done}/${SESSIONS}`);+ }+ }+ );++ const uniq = new Set(sessions.map((s) => s.sessionId)).size;+ check('all sessions sent without HTTP errors', errors === 0, `${errors} errors`);+ check(+ 'each session got a distinct session id',+ uniq === SESSIONS,+ `${uniq} unique / ${sessions.length} ok / ${SESSIONS} sent`+ );+ console.log(` emit took ${((Date.now() - started) / 1000).toFixed(1)}s`);+ return sessions;+}++async function settle(sessions: Session[]) {+ scenario('settle: all sessions opened in ClickHouse');+ const ids = sessions.map((s) => s.sessionId);+ const starts = await pollUntil(+ async () => {+ const c = await countByName(ids, 'session_start');+ console.log(` …session_start ${c}/${ids.length}`);+ return c >= ids.length ? c : null;+ },+ { timeoutMs: 60_000, intervalMs: 2000 }+ );+ check('clickhouse: one session_start per session', starts === ids.length, `got ${starts}`);+}++async function drain(sessions: Session[]) {+ scenario('drain: reap + flush until nothing remains');+ console.log(` …waiting ${IDLE_WAIT_MS}ms for all sessions to idle out`);+ await sleep(IDLE_WAIT_MS);++ const ids = sessions.map((s) => s.sessionId);+ const target = ids.length;+ const deadline = Date.now() + DRAIN_TIMEOUT_MS;+ let ends = 0;+ let remaining = Number.POSITIVE_INFINITY;++ while (Date.now() < deadline) {+ await triggerReaper(); // enqueue session_end for idle sessions+ await sleep(1000); // let the worker process the close jobs+ await triggerCron('flushEvents'); // push session_start/end + events → CH+ await triggerCron('flushSessions'); // push sessions table rows → CH++ ends = await countByName(ids, 'session_end');+ remaining = await redis.zcard(wallclockKey);+ console.log(` …session_end ${ends}/${target}, wallclock remaining ${remaining}`);+ if (ends >= target && remaining === 0) break;+ await sleep(1500);+ }++ check('clickhouse: one session_end per session', ends === target, `got ${ends}`);+ check('redis: wallclock index fully drained', remaining === 0, `${remaining} left`);++ const bufLen = await redis.llen(SESSION_BUFFER_LIST);+ check('redis: session buffer drained to ClickHouse', bufLen === 0, `${bufLen} rows pending`);++ // Spot-check a sample of devices: no session blob should linger after close.+ const sample = sessions.filter((_, k) => k % Math.ceil(sessions.length / 20) === 0);+ let leaked = 0;+ for (const s of sample) {+ if (await getBlob(s.deviceId)) leaked++;+ }+ check('redis: no session blobs leaked (sampled)', leaked === 0, `${leaked}/${sample.length} leaked`);+}++async function reconcile(sessions: Session[]) {+ scenario('reconcile: ClickHouse event counts');+ const ids = sessions.map((s) => s.sessionId);+ const n = ids.length;+ const inList = ids.map((id) => `'${id}'`).join(',');+ const rows = await chQuery<{ name: string; c: string }>(+ `SELECT name, count() AS c FROM events WHERE project_id = '${PROJECT_ID}' AND session_id IN (${inList}) GROUP BY name ORDER BY c DESC`+ );+ const byName = new Map(rows.map((r) => [r.name, Number(r.c)]));+ const total = [...byName.values()].reduce((a, b) => a + b, 0);+ console.log(` event breakdown: ${rows.map((r) => `${r.name}=${r.c}`).join(' ')}`);++ const starts = byName.get('session_start') ?? 0;+ const ends = byName.get('session_end') ?? 0;+ // Each session = N events we sent + the synthetic session_start + session_end.+ check('session_start == sessions', starts === n, `${starts} vs ${n}`);+ check('session_end == sessions', ends === n, `${ends} vs ${n}`);+ check(+ 'total events == sessions × (events + start + end)',+ total === n * (EVENTS_PER_SESSION + 2),+ `${total} vs ${n * (EVENTS_PER_SESSION + 2)}`+ );+}++async function main() {+ console.log(+ `Session STRESS — api=${API_URL} worker=${WORKER_URL} timeout=${SESSION_TIMEOUT_MS}ms ` ++ `sessions=${SESSIONS} events=${EVENTS_PER_SESSION} concurrency=${CONCURRENCY}`+ );+ await preflight();+ await ensureFixtures();++ const sessions = await emit();+ await settle(sessions);+ await drain(sessions);+ await reconcile(sessions);++ await shutdown(summarize());+}++main().catch(async (error) => {+ console.error('\nFATAL:', error);+ await shutdown(1);+});diff --git a/apps/api/package.json b/apps/api/package.jsonindex fc6ef7d9a..d3c73d376 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -9,6 +9,8 @@"build": "rm -rf dist && tsdown","gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts","test:manage": "jiti scripts/test-manage-api.ts",+ "e2e:sessions": "LOG_LEVEL=warn dotenv -e ../../.env -- jiti e2e/session-e2e.ts",+ "e2e:sessions:stress": "LOG_LEVEL=warn dotenv -e ../../.env -- jiti e2e/session-stress.ts","test": "vitest","test:run": "vitest run","typecheck": "tsc --noEmit"diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 2cda17f2a..3b2c14615 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -1,10 +1,13 @@import { generateId } from '@openpanel/common';import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';import {+ convertClickhouseDateToJs,getProfileById,getSalts,groupBuffer,replayBuffer,+ SESSION_TIMEOUT_MS,+ sessionBuffer,upsertProfile,} from '@openpanel/db';import { type GeoLocation, getGeoLocation } from '@openpanel/geo';@@ -14,7 +17,6 @@ import {produceIncomingEvent,shouldUseKafka,} from '@openpanel/queue';-import { getRedisCache } from '@openpanel/redis';import type {IAssignGroupPayload,IDecrementPayload,@@ -182,6 +184,7 @@ async function buildContext(ua,salts,overrideDeviceId,+ eventTimeMs: timestamp.timestamp,});return {@@ -486,32 +489,38 @@ export async function fetchDeviceId(});try {- const multi = getRedisCache().multi();- multi.hget(- `bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`,- 'data'- );- multi.hget(- `bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`,- 'data'- );- const res = await multi.exec();- if (res?.[0]?.[1]) {- const data = JSON.parse(res?.[0]?.[1] as string);- const sessionId = data.payload.sessionId;+ const [current, previous] = await Promise.all([+ sessionBuffer.getExistingSession({+ projectId,+ deviceId: currentDeviceId,+ }),+ sessionBuffer.getExistingSession({+ projectId,+ deviceId: previousDeviceId,+ }),+ ]);++ // Blob has no TTL — only treat the session as "current" if its last+ // event is within the idle window. Otherwise the SDK should ask the+ // server to start a fresh session id on the next event.+ const now = Date.now();+ const isLive = (s: typeof current) =>+ !!s &&+ now - convertClickhouseDateToJs(s.ended_at).getTime() <+ SESSION_TIMEOUT_MS;++ if (current && isLive(current)) {return reply.status(200).send({deviceId: currentDeviceId,- sessionId,+ sessionId: current.id,message: 'current session exists for this device id',});}- if (res?.[1]?.[1]) {- const data = JSON.parse(res?.[1]?.[1] as string);- const sessionId = data.payload.sessionId;+ if (previous && isLive(previous)) {return reply.status(200).send({deviceId: previousDeviceId,- sessionId,+ sessionId: previous.id,message: 'previous session exists for this device id',});}diff --git a/apps/api/src/utils/ids.test.ts b/apps/api/src/utils/ids.test.tsindex c1a4cf43c..e6d8e952c 100644--- a/apps/api/src/utils/ids.test.ts+++ b/apps/api/src/utils/ids.test.ts@@ -1,91 +1,74 @@/**- * Tests for getDeviceId — specifically the caller-supplied (override) device id- * path. The override must resolve a real, stable sessionId through the same- * `getInfoFromSession` logic internal device ids use, so sessions open/extend/close- * identically. Returning an empty sessionId here is the bug that left sessions- * unbuffered and made the createSessionEnd job throw "Session not found".+ * Tests for getDeviceId's session resolution against the device-keyed session+ * store. The id is reused from the live blob only while it's within the idle+ * window; a lingering (past-window) blob must NOT be reused — see the guard in+ * ids.ts. Override device ids resolve through the same path with a single read.*- * Redis is mocked: we drive `multi().exec()` to simulate "an active sessionEnd job- * exists" vs "no active job" without a real connection.+ * We spy on the real `sessionBuffer.getExistingSession` singleton (no Redis),+ * and use the real `formatClickhouseDate` so `ended_at` round-trips exactly.*/-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';--const { execMock } = vi.hoisted(() => ({ execMock: vi.fn() }));--vi.mock('@openpanel/redis', async (importOriginal) => {- const actual = await importOriginal<typeof import('@openpanel/redis')>();- return {- ...actual,- getRedisCache: () => ({- multi: () => {- const chain = {- hget: () => chain,- exec: execMock,- };- return chain;- },- }),- };-});--const { getDeviceId } = await import('./ids');+import type { IClickhouseSession } from '@openpanel/db';+import { formatClickhouseDate, sessionBuffer } from '@openpanel/db';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { getDeviceId } from './ids';+const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();+const MINUTE = 60 * 1000;const SALTS = { current: 'salt-current', previous: 'salt-previous' };const BASE = {projectId: 'proj-1',ip: '1.2.3.4',ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/148.0.0.0',salts: SALTS,+ eventTimeMs: NOW,};-beforeEach(() => {- vi.useFakeTimers();- vi.setSystemTime(new Date('2026-06-08T12:00:00.000Z'));- execMock.mockReset();-});+// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.+const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>+ ({+ id,+ ended_at: formatClickhouseDate(new Date(endedAtMs)),+ }) as unknown as IClickhouseSession;afterEach(() => {- vi.useRealTimers();+ vi.restoreAllMocks();});-describe('getDeviceId — override device id', () => {- it('derives a non-empty, deterministic sessionId when no active session exists', async () => {- execMock.mockResolvedValue([- [null, null],- [null, null],- ]);+describe('getDeviceId — session resolution', () => {+ it('mints a deterministic, stable id when no session exists', async () => {+ const spy = vi+ .spyOn(sessionBuffer, 'getExistingSession')+ .mockResolvedValue(null);- const result = await getDeviceId({- ...BASE,- overrideDeviceId: 'cookie-abc',- });+ const a = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });+ const b = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });- expect(result.deviceId).toBe('cookie-abc');- expect(result.sessionId).toBeTruthy();+ expect(a.sessionId).toBeTruthy();+ expect(b.sessionId).toBe(a.sessionId); // same window → same id+ expect(a.deviceId).toBe('cookie-abc');+ expect(spy).toHaveBeenCalled();+ });- // Same override within the same time window → same session id (stable).- const again = await getDeviceId({+ it('reuses the live session id when within the idle window', async () => {+ vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(+ fakeSession('sess-live', NOW - MINUTE)+ );++ const result = await getDeviceId({...BASE,overrideDeviceId: 'cookie-abc',});- expect(again.sessionId).toBe(result.sessionId);++ expect(result).toEqual({ deviceId: 'cookie-abc', sessionId: 'sess-live' });});- it('reuses the sessionId from an active sessionEnd job instead of splitting', async () => {- execMock.mockResolvedValue([- [- null,- JSON.stringify({- payload: {- projectId: 'proj-1',- deviceId: 'cookie-abc',- sessionId: 'sess-active-123',- },- }),- ],- [null, null],- ]);+ it('does NOT reuse a session that has lingered past the idle window', async () => {+ vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(+ // ended 31 min ago — past the 30 min timeout; the reaper just hasn't+ // closed it yet (blobs have no TTL).+ fakeSession('sess-stale', NOW - 31 * MINUTE)+ );const result = await getDeviceId({...BASE,@@ -93,20 +76,35 @@ describe('getDeviceId — override device id', () => {});expect(result.deviceId).toBe('cookie-abc');- expect(result.sessionId).toBe('sess-active-123');+ expect(result.sessionId).toBeTruthy();+ expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one});- it('produces distinct session ids for distinct override device ids', async () => {- execMock.mockResolvedValue([- [null, null],- [null, null],- ]);+ it('reads the store once for an override (no redundant previous lookup)', async () => {+ const spy = vi+ .spyOn(sessionBuffer, 'getExistingSession')+ .mockResolvedValue(null);- const a = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-a' });- const b = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-b' });+ await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });- expect(a.sessionId).toBeTruthy();- expect(b.sessionId).toBeTruthy();- expect(a.sessionId).not.toBe(b.sessionId);+ expect(spy).toHaveBeenCalledTimes(1);+ expect(spy).toHaveBeenCalledWith({+ projectId: 'proj-1',+ deviceId: 'cookie-abc',+ });+ });++ it('checks both current and previous salt windows for internal ids', async () => {+ const spy = vi+ .spyOn(sessionBuffer, 'getExistingSession')+ .mockResolvedValue(null);++ await getDeviceId(BASE); // no override → IP+UA hashing++ expect(spy).toHaveBeenCalledTimes(2);+ const deviceIds = spy.mock.calls.map((c) =>+ 'deviceId' in c[0] ? c[0].deviceId : ''+ );+ expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes});});diff --git a/apps/api/src/utils/ids.ts b/apps/api/src/utils/ids.tsindex ec0cfc993..21611ac38 100644--- a/apps/api/src/utils/ids.ts+++ b/apps/api/src/utils/ids.ts@@ -1,8 +1,11 @@import crypto from 'node:crypto';import { generateDeviceId } from '@openpanel/common/server';-import { getSafeJson } from '@openpanel/json';-import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';-import { getRedisCache } from '@openpanel/redis';+import {+ convertClickhouseDateToJs,+ SESSION_TIMEOUT_MS,+ sessionBuffer,+} from '@openpanel/db';+import type { IClickhouseSession } from '@openpanel/db';export async function getDeviceId({projectId,@@ -10,20 +13,24 @@ export async function getDeviceId({ua,salts,overrideDeviceId,+ eventTimeMs,}: {projectId: string;ip: string;ua: string | undefined;salts: { current: string; previous: string };overrideDeviceId?: string;+ /** Event timestamp (ms). Used to decide whether an existing session is+ * still within its idle window. Defaults to `Date.now()`. */+ eventTimeMs?: number;}) {if (overrideDeviceId) {- // Resolve a caller-supplied device id through the same path as internal ones- // (stable, so it's both candidates) — sessions open/extend/close identically.+ // A caller-supplied device id is stable (no salt rotation), so it's the only+ // candidate — resolve it through the same path as internal ids.return await getInfoFromSession({projectId,- currentDeviceId: overrideDeviceId,- previousDeviceId: overrideDeviceId,+ deviceIds: [overrideDeviceId],+ eventTimeMs: eventTimeMs ?? Date.now(),});}@@ -46,8 +53,8 @@ export async function getDeviceId({return await getInfoFromSession({projectId,- currentDeviceId,- previousDeviceId,+ deviceIds: [currentDeviceId, previousDeviceId],+ eventTimeMs: eventTimeMs ?? Date.now(),});}@@ -56,59 +63,74 @@ interface DeviceIdResult {sessionId: string;}+/**+ * Returns true when an existing session is recent enough that the incoming+ * event should EXTEND it rather than start a new session.+ *+ * Critical: blobs no longer have a Redis TTL (so the reaper can always find+ * them), which means an existing session blob may linger past its idle+ * window. We must NOT blindly reuse `existing.id` — if we did, the worker's+ * boundary detection would open a "new" session with the same id as the+ * closed one, breaking the id-based extension check in createSessionEnd.+ */+function withinIdleWindow(+ session: IClickhouseSession,+ eventTimeMs: number+): boolean {+ const lastEventMs = convertClickhouseDateToJs(session.ended_at).getTime();+ return eventTimeMs - lastEventMs < SESSION_TIMEOUT_MS;+}+async function getInfoFromSession({projectId,- currentDeviceId,- previousDeviceId,+ deviceIds,+ eventTimeMs,}: {projectId: string;- currentDeviceId: string;- previousDeviceId: string;+ /** Candidate device ids in priority order (e.g. [current, previous] salt+ * windows, or just [override]). Deduped; the first is canonical. */+ deviceIds: string[];+ eventTimeMs: number;}): Promise<DeviceIdResult> {+ const candidates = [...new Set(deviceIds.filter(Boolean))];+ const primary = candidates[0] ?? '';+try {- const multi = getRedisCache().multi();- multi.hget(- `bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`,- 'data'- );- multi.hget(- `bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`,- 'data'+ // Reading the live blob is the source of truth for an active session — it's+ // what keeps a visit on one id across page reloads and salt rotation. Don't+ // drop this read to "save" a lookup or sessions split at the bucket boundary.+ const sessions = await Promise.all(+ candidates.map((deviceId) =>+ sessionBuffer.getExistingSession({ projectId, deviceId })+ ));- const res = await multi.exec();- if (res?.[0]?.[1]) {- const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(- (res?.[0]?.[1] as string) ?? ''- );- if (data) {- return {- deviceId: currentDeviceId,- sessionId: data.payload.sessionId,- };- }- }- if (res?.[1]?.[1]) {- const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(- (res?.[1]?.[1] as string) ?? ''- );- if (data) {- return {- deviceId: previousDeviceId,- sessionId: data.payload.sessionId,- };++ for (const [i, session] of sessions.entries()) {+ if (session && withinIdleWindow(session, eventTimeMs)) {+ return { deviceId: candidates[i]!, sessionId: session.id };}}} catch (error) {- console.error('Error getting session end GET /track/device-id', error);+ console.error('Error resolving session for device id', error);}return {- deviceId: currentDeviceId,+ deviceId: primary,+ // Deterministic id for the first event of a session and to bridge the window+ // before the worker persists the blob (API resolves synchronously, worker+ // writes async — same bucket → same id, so they agree).+ //+ // The bucket window MUST track the idle timeout: a gap > the window has to+ // land in a new bucket so a boundary mints a *fresh* id. If it didn't (e.g.+ // a hardcoded 30min while SESSION_TIMEOUT_MS is shorter), the worker would+ // reopen the just-closed id and its session_end would be skipped. Grace must+ // stay < window or getSessionId throws.sessionId: getSessionId({projectId,- deviceId: currentDeviceId,- graceMs: 5 * 1000,- windowMs: 1000 * 60 * 30,+ deviceId: primary,+ eventMs: eventTimeMs,+ graceMs: Math.min(5_000, Math.floor(SESSION_TIMEOUT_MS / 6)),+ windowMs: SESSION_TIMEOUT_MS,}),};}diff --git a/apps/worker/src/boot-cron.ts b/apps/worker/src/boot-cron.tsindex d2458076e..4150308dc 100644--- a/apps/worker/src/boot-cron.ts+++ b/apps/worker/src/boot-cron.ts@@ -93,6 +93,16 @@ export async function bootCron() {type: 'cohortRefresh',pattern: '*/30 * * * *',},+ {+ name: 'sessionReaper',+ type: 'sessionReaper',+ pattern: 1000 * 60 * 5, // every 5 minutes+ },+ {+ name: 'sessionVacuum',+ type: 'sessionVacuum',+ pattern: '0 4 * * *', // daily at 04:00 UTC — backstop for cleanup leaks+ },];if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {diff --git a/apps/worker/src/boot-debug.ts b/apps/worker/src/boot-debug.tsindex 58a7d6a84..a95c5cac1 100644--- a/apps/worker/src/boot-debug.ts+++ b/apps/worker/src/boot-debug.ts@@ -18,6 +18,8 @@ const CRON_TYPES = ['onboarding','gscSync','cohortRefresh',+ 'sessionReaper',+ 'sessionVacuum',] as const satisfies readonly CronQueueType[];function escapeHtml(value: string) {diff --git a/apps/worker/src/jobs/cron.session-reaper.test.ts b/apps/worker/src/jobs/cron.session-reaper.test.tsnew file mode 100644index 000000000..9d6423f40--- /dev/null+++ b/apps/worker/src/jobs/cron.session-reaper.test.ts@@ -0,0 +1,119 @@+/**+ * Tests for the session reaper — the wall-clock deadman that closes idle+ * sessions. Redis is mocked (we drive the wallclock ZSET + lock), sessionBuffer+ * and enqueueSessionEndV2 are mocked so we assert decisions, not side effects.+ */++import type { IClickhouseSession } from '@openpanel/db';+import { sessionBuffer } from '@openpanel/db';+import { beforeEach, describe, expect, it, vi } from 'vitest';+import { enqueueSessionEndV2 } from '@/utils/session-handler';+import { sessionReaperCronJob } from './cron.session-reaper';++const { redisMock } = vi.hoisted(() => ({+ redisMock: {+ smembers: vi.fn(),+ set: vi.fn(),+ del: vi.fn(),+ zrangebyscore: vi.fn(),+ zcard: vi.fn(),+ srem: vi.fn(),+ zrem: vi.fn(),+ },+}));++vi.mock('@openpanel/redis', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@openpanel/redis')>();+ return { ...actual, getRedisCache: () => redisMock };+});++vi.mock('@openpanel/db', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@openpanel/db')>();+ return { ...actual, sessionBuffer: { getExistingSession: vi.fn() } };+});++vi.mock('@/utils/session-handler', () => ({+ enqueueSessionEndV2: vi.fn().mockResolvedValue(undefined),+}));++// Concrete metric stubs — avoid loading the real metrics module (it registers+// prom gauges at import) and avoid a Proxy (its `then` trap confuses ESM interop).+vi.mock('@/metrics', () => {+ const counter = () => ({ inc: vi.fn(), observe: vi.fn(), set: vi.fn() });+ return {+ sessionEndsEnqueued: counter(),+ sessionsReaped: counter(),+ sessionsReaperOrphans: counter(),+ };+});++const getExistingSession = vi.mocked(sessionBuffer.getExistingSession);++const session = (id: string, deviceId: string): IClickhouseSession =>+ ({+ id,+ project_id: 'proj-1',+ device_id: deviceId,+ profile_id: deviceId,+ ended_at: '2026-06-08 11:00:00',+ duration: 1000,+ groups: [],+ }) as unknown as IClickhouseSession;++beforeEach(() => {+ vi.clearAllMocks();+ redisMock.smembers.mockResolvedValue(['proj-1']);+ redisMock.set.mockResolvedValue('OK'); // lock acquired+ redisMock.del.mockResolvedValue(1);+ redisMock.zcard.mockResolvedValue(0); // project drained after reap+ redisMock.srem.mockResolvedValue(1);+ redisMock.zrem.mockResolvedValue(1);+ redisMock.zrangebyscore.mockResolvedValue([]);+});++describe('sessionReaperCronJob', () => {+ it('enqueues a session_end for an idle session blob', async () => {+ redisMock.zrangebyscore.mockResolvedValue(['dev-1']);+ getExistingSession.mockResolvedValue(session('sess-1', 'dev-1'));++ await sessionReaperCronJob();++ expect(enqueueSessionEndV2).toHaveBeenCalledTimes(1);+ expect(+ vi.mocked(enqueueSessionEndV2).mock.calls[0]![0].closedSession.id+ ).toBe('sess-1');+ // wallclock index emptied → project removed from the active set+ expect(redisMock.srem).toHaveBeenCalledWith('session:projects', 'proj-1');+ });++ it('ZREMs an orphan (wallclock entry with no blob) and does not enqueue', async () => {+ redisMock.zrangebyscore.mockResolvedValue(['dev-orphan']);+ getExistingSession.mockResolvedValue(null);++ await sessionReaperCronJob();++ expect(enqueueSessionEndV2).not.toHaveBeenCalled();+ expect(redisMock.zrem).toHaveBeenCalledWith(+ 'session:wallclock:proj-1',+ 'dev-orphan'+ );+ });++ it('skips a project when the advisory lock is held', async () => {+ redisMock.set.mockResolvedValue(null); // NX failed → another pod owns it++ await sessionReaperCronJob();++ expect(redisMock.zrangebyscore).not.toHaveBeenCalled();+ expect(enqueueSessionEndV2).not.toHaveBeenCalled();+ });++ it('is a no-op when disabled via SESSION_REAPER=0', async () => {+ vi.stubEnv('SESSION_REAPER', '0');++ await sessionReaperCronJob();++ expect(redisMock.smembers).not.toHaveBeenCalled();+ vi.unstubAllEnvs();+ });+});diff --git a/apps/worker/src/jobs/cron.session-reaper.ts b/apps/worker/src/jobs/cron.session-reaper.tsnew file mode 100644index 000000000..867b07db9--- /dev/null+++ b/apps/worker/src/jobs/cron.session-reaper.ts@@ -0,0 +1,204 @@+import { SESSION_TIMEOUT_MS, sessionBuffer } from '@openpanel/db';+import { getRedisCache } from '@openpanel/redis';+import { logger as baseLogger } from '@/utils/logger';+import {+ sessionEndsEnqueued,+ sessionsReaped,+ sessionsReaperOrphans,+} from '@/metrics';+import { enqueueSessionEndV2 } from '@/utils/session-handler';++const logger = baseLogger.child({ job: 'session-reaper' });++const PROJECTS_SET_KEY = 'session:projects';+const REAPER_BATCH_SIZE = Number.parseInt(+ process.env.SESSION_REAPER_BATCH_SIZE || '5000',+ 10+);+// Wall-clock deadman: close sessions whose last *received* event is older+// than this. Single source of truth for "this session has ended", regardless+// of project traffic.+//+// Default 30min, matching SESSION_TIMEOUT_MS. With the 5-min reaper interval,+// max session_end latency is 30-35min after the last event.+//+// During a worker pause longer than this, sessions in flight will be split+// when the worker resumes. Bump via env if you expect long pauses, but+// remember that BullMQ jobId-dedup + the extension check in createSessionEnd+// already cover normal queue lag.+const WALLCLOCK_DEADMAN_MS = Number.parseInt(+ process.env.SESSION_REAPER_WALLCLOCK_DEADMAN_MS ||+ String(SESSION_TIMEOUT_MS),+ 10+);+const LOCK_TTL_SECONDS = 60;++const wallclockSetKey = (projectId: string) =>+ `session:wallclock:${projectId}`;+const lockKey = (projectId: string) => `session:reaper:lock:${projectId}`;++/**+ * Cron: scan every project with active sessions, close any whose last event+ * was received more than the deadman ago in wall-clock time.+ *+ * Idempotent — uses BullMQ jobId-dedup via enqueueSessionEndV2.+ */+export async function sessionReaperCronJob() {+ if (process.env.SESSION_REAPER === '0') {+ return;+ }++ const redis = getRedisCache();+ const projectIds = await redis.smembers(PROJECTS_SET_KEY);++ if (projectIds.length === 0) {+ return;+ }++ logger.debug({ projectCount: projectIds.length }, 'Reaper tick');++ let totalReaped = 0;+ let totalErrors = 0;++ for (const projectId of projectIds) {+ try {+ const reaped = await reapProject(projectId);+ totalReaped += reaped;+ } catch (error) {+ totalErrors++;+ logger.error({ err: error, projectId }, 'Failed to reap project');+ }+ }++ if (totalReaped > 0 || totalErrors > 0) {+ logger.info(+ { reaped: totalReaped, errors: totalErrors, projects: projectIds.length },+ 'Reaper tick complete',+ );+ }+}++async function reapProject(projectId: string): Promise<number> {+ const redis = getRedisCache();++ // Per-project advisory lock keeps multiple worker pods from reaping the+ // same project simultaneously. Set NX with a short TTL; the lock is best+ // effort — if it expires mid-tick, BullMQ jobId-dedup is the real guard.+ const locked = await redis.set(+ lockKey(projectId),+ '1',+ 'EX',+ LOCK_TTL_SECONDS,+ 'NX'+ );+ if (locked === null) {+ return 0;+ }++ try {+ const now = Date.now();+ const cutoff = now - WALLCLOCK_DEADMAN_MS;++ const candidates = await redis.zrangebyscore(+ wallclockSetKey(projectId),+ 0,+ cutoff,+ 'LIMIT',+ 0,+ REAPER_BATCH_SIZE+ );++ let reaped = 0;+ for (const deviceId of candidates) {+ if (await closeSession(projectId, deviceId)) {+ reaped++;+ }+ }++ // If the project has no remaining sessions in the wallclock index,+ // remove it from the projects set so we stop iterating it.+ const remaining = await redis.zcard(wallclockSetKey(projectId));+ if (remaining === 0) {+ await redis.srem(PROJECTS_SET_KEY, projectId);+ }++ return reaped;+ } finally {+ await redis.del(lockKey(projectId));+ }+}++async function closeSession(+ projectId: string,+ deviceId: string+): Promise<boolean> {+ const session = await sessionBuffer.getExistingSession({+ projectId,+ deviceId,+ });++ if (!session) {+ // Sorted-set entry exists but the blob is gone. Without a TTL on the+ // blob this should be rare — only happens if cleanup() partially+ // failed (worker crash mid-MULTI, network partition). Log + count it+ // and ZREM the orphan entry.+ sessionsReaperOrphans.inc({ reason: 'deadman' });+ baseLogger.warn(+ { projectId, deviceId },+ 'Reaper found wallclock entry without blob — likely a partial cleanup failure',+ );+ await getRedisCache().zrem(wallclockSetKey(projectId), deviceId);+ return false;+ }++ try {+ await enqueueSessionEndV2({+ payload: {+ projectId: session.project_id,+ deviceId: session.device_id,+ sessionId: session.id,+ profileId: session.profile_id ?? '',+ name: 'session_end',+ properties: {},+ groups: session.groups ?? [],+ createdAt: new Date(session.ended_at),+ duration: session.duration ?? 0,+ sdkName: '',+ sdkVersion: '',+ city: session.city ?? '',+ country: session.country ?? '',+ region: session.region ?? '',+ longitude: session.longitude ?? undefined,+ latitude: session.latitude ?? undefined,+ path: session.exit_path ?? '',+ origin: session.exit_origin ?? '',+ referrer: session.referrer ?? '',+ referrerName: session.referrer_name ?? '',+ referrerType: session.referrer_type ?? '',+ os: session.os ?? '',+ osVersion: session.os_version ?? '',+ browser: session.browser ?? '',+ browserVersion: session.browser_version ?? '',+ device: session.device ?? '',+ brand: session.brand ?? '',+ model: session.model ?? '',+ },+ closedSession: session,+ });++ sessionsReaped.inc({ reason: 'deadman' });+ sessionEndsEnqueued.inc({ source: 'reaper' });++ logger.debug(+ { sessionId: session.id, projectId, deviceId },+ 'Enqueued session_end (reaped)',+ );+ return true;+ } catch (error) {+ logger.error(+ { err: error, sessionId: session.id, projectId, deviceId },+ 'Failed to enqueue session_end during reap',+ );+ return false;+ }+}diff --git a/apps/worker/src/jobs/cron.session-vacuum.test.ts b/apps/worker/src/jobs/cron.session-vacuum.test.tsnew file mode 100644index 000000000..419dcdfc9--- /dev/null+++ b/apps/worker/src/jobs/cron.session-vacuum.test.ts@@ -0,0 +1,90 @@+/**+ * Tests for the daily vacuum — the backstop for sessions whose cleanup() leaked+ * (worker crash mid-MULTI). For wallclock entries older than the stale threshold:+ * a lingering blob → id-gated cleanup(); a missing blob → ZREM the orphan.+ */++import type { IClickhouseSession } from '@openpanel/db';+import { sessionBuffer } from '@openpanel/db';+import { beforeEach, describe, expect, it, vi } from 'vitest';+import { sessionVacuumCronJob } from './cron.session-vacuum';++const { redisMock } = vi.hoisted(() => ({+ redisMock: {+ smembers: vi.fn(),+ zrangebyscore: vi.fn(),+ zrem: vi.fn(),+ },+}));++vi.mock('@openpanel/redis', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@openpanel/redis')>();+ return { ...actual, getRedisCache: () => redisMock };+});++vi.mock('@openpanel/db', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@openpanel/db')>();+ return {+ ...actual,+ sessionBuffer: { getExistingSession: vi.fn(), cleanup: vi.fn() },+ };+});++vi.mock('@/metrics', () => ({+ sessionsVacuumed: { inc: vi.fn() },+}));++const getExistingSession = vi.mocked(sessionBuffer.getExistingSession);+const cleanup = vi.mocked(sessionBuffer.cleanup);++beforeEach(() => {+ vi.clearAllMocks();+ redisMock.smembers.mockResolvedValue(['proj-1']);+ redisMock.zrem.mockResolvedValue(1);+ redisMock.zrangebyscore.mockResolvedValue([]);+ cleanup.mockResolvedValue(undefined as never);+});++describe('sessionVacuumCronJob', () => {+ it('id-gated cleanup() for a stale blob that lingered', async () => {+ redisMock.zrangebyscore.mockResolvedValue(['dev-stale']);+ getExistingSession.mockResolvedValue({+ id: 'sess-stale',+ project_id: 'proj-1',+ device_id: 'dev-stale',+ profile_id: 'dev-stale',+ } as unknown as IClickhouseSession);++ await sessionVacuumCronJob();++ expect(cleanup).toHaveBeenCalledWith({+ projectId: 'proj-1',+ deviceId: 'dev-stale',+ sessionId: 'sess-stale',+ profileId: 'dev-stale',+ });+ expect(redisMock.zrem).not.toHaveBeenCalled();+ });++ it('ZREMs an orphan wallclock entry when the blob is gone', async () => {+ redisMock.zrangebyscore.mockResolvedValue(['dev-orphan']);+ getExistingSession.mockResolvedValue(null);++ await sessionVacuumCronJob();++ expect(cleanup).not.toHaveBeenCalled();+ expect(redisMock.zrem).toHaveBeenCalledWith(+ 'session:wallclock:proj-1',+ 'dev-orphan'+ );+ });++ it('is a no-op when disabled via SESSION_VACUUM=0', async () => {+ vi.stubEnv('SESSION_VACUUM', '0');++ await sessionVacuumCronJob();++ expect(redisMock.smembers).not.toHaveBeenCalled();+ vi.unstubAllEnvs();+ });+});diff --git a/apps/worker/src/jobs/cron.session-vacuum.ts b/apps/worker/src/jobs/cron.session-vacuum.tsnew file mode 100644index 000000000..ea1911395--- /dev/null+++ b/apps/worker/src/jobs/cron.session-vacuum.ts@@ -0,0 +1,101 @@+import { sessionBuffer } from '@openpanel/db';+import { getRedisCache } from '@openpanel/redis';+import { logger as baseLogger } from '@/utils/logger';+import { sessionsVacuumed } from '@/metrics';++const logger = baseLogger.child({ job: 'session-vacuum' });++const PROJECTS_SET_KEY = 'session:projects';+const VACUUM_BATCH_SIZE = Number.parseInt(+ process.env.SESSION_VACUUM_BATCH_SIZE || '1000',+ 10+);+// Anything in the wallclock index that hasn't moved in this long is stale —+// either a leaked entry (cleanup() partially failed) or a session that's+// been forgotten by every code path. Much larger than the reaper deadman+// so it never races with normal reap timing.+const VACUUM_STALE_THRESHOLD_MS = Number.parseInt(+ process.env.SESSION_VACUUM_STALE_THRESHOLD_MS ||+ String(1000 * 60 * 60 * 24 * 7), // 7 days+ 10+);++const wallclockSetKey = (projectId: string) =>+ `session:wallclock:${projectId}`;++/**+ * Daily backstop for the rare case where `cleanup()` fails to fully delete+ * a session (worker crash mid-MULTI, Redis partition). Scans each project's+ * wallclock index for entries older than the stale threshold and:+ *+ * - If the blob still exists → run cleanup() (id-gated, atomic via Lua)+ * - If the blob is gone → ZREM the orphan entry+ *+ * Both paths increment `sessions_vacuumed_total` with a reason label, so+ * persistent non-zero values surface deeper issues.+ */+export async function sessionVacuumCronJob() {+ if (process.env.SESSION_VACUUM === '0') {+ return;+ }++ const redis = getRedisCache();+ const projectIds = await redis.smembers(PROJECTS_SET_KEY);++ if (projectIds.length === 0) {+ return;+ }++ logger.info({ projectCount: projectIds.length }, 'Vacuum tick starting');++ const cutoff = Date.now() - VACUUM_STALE_THRESHOLD_MS;+ let total = 0;+ let staleBlobs = 0;+ let missingBlobs = 0;++ for (const projectId of projectIds) {+ try {+ const candidates = await redis.zrangebyscore(+ wallclockSetKey(projectId),+ 0,+ cutoff,+ 'LIMIT',+ 0,+ VACUUM_BATCH_SIZE+ );++ for (const deviceId of candidates) {+ const session = await sessionBuffer.getExistingSession({+ projectId,+ deviceId,+ });++ if (session) {+ // Blob still exists but is ancient — cleanup leaked. Use the+ // id-gated cleanup so we don't race with a fresh session that+ // happens to occupy the same slot.+ await sessionBuffer.cleanup({+ projectId,+ deviceId,+ sessionId: session.id,+ profileId: session.profile_id,+ });+ sessionsVacuumed.inc({ reason: 'stale_blob' });+ staleBlobs++;+ } else {+ // Orphan: blob is gone, wallclock entry left behind.+ await redis.zrem(wallclockSetKey(projectId), deviceId);+ sessionsVacuumed.inc({ reason: 'missing_blob' });+ missingBlobs++;+ }+ total++;+ }+ } catch (error) {+ logger.error({ err: error, projectId }, 'Vacuum failed for project');+ }+ }++ if (total > 0) {+ logger.info({ total, staleBlobs, missingBlobs }, 'Vacuum tick complete');+ }+}diff --git a/apps/worker/src/jobs/cron.ts b/apps/worker/src/jobs/cron.tsindex 305c30471..43504b361 100644--- a/apps/worker/src/jobs/cron.ts+++ b/apps/worker/src/jobs/cron.ts@@ -1,17 +1,26 @@-import type { Job } from 'bullmq';--import { eventBuffer, groupBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';+import {+ eventBuffer,+ groupBuffer,+ profileBackfillBuffer,+ profileBuffer,+ replayBuffer,+ sessionBuffer,+} from '@openpanel/db';import type { CronQueuePayload } from '@openpanel/queue';-+import type { Job } from 'bullmq';import { cohortRefreshCronJob } from './cron.cohort-refresh';import { jobDelete } from './cron.delete';-import { gscSyncAllJob } from './gsc';import { onboardingJob } from './cron.onboarding';import { ping } from './cron.ping';import { salt } from './cron.salt';+import { sessionReaperCronJob } from './cron.session-reaper';+import { sessionVacuumCronJob } from './cron.session-vacuum';+import { gscSyncAllJob } from './gsc';import { insightsDailyJob } from './insights';+import { logger } from '@/utils/logger';export async function cronJob(job: Job<CronQueuePayload>) {+ logger.debug(`Cron job started - ${job.data.type}`);switch (job.data.type) {case 'salt': {return await salt();@@ -52,5 +61,11 @@ export async function cronJob(job: Job<CronQueuePayload>) {case 'cohortRefresh': {return await cohortRefreshCronJob();}+ case 'sessionReaper': {+ return await sessionReaperCronJob();+ }+ case 'sessionVacuum': {+ return await sessionVacuumCronJob();+ }}}diff --git a/apps/worker/src/jobs/events.create-session-end.test.ts b/apps/worker/src/jobs/events.create-session-end.test.tsnew file mode 100644index 000000000..cdb71d1d2--- /dev/null+++ b/apps/worker/src/jobs/events.create-session-end.test.ts@@ -0,0 +1,153 @@+/**+ * Tests for createSessionEnd — the close job. Covers the three-case logic+ * (extended-after-enqueue skip, live vs snapshot, boundary), the first-writer+ * idempotency claim, and that cleanup() is id-gated to the closed session.+ */++import {+ createEvent,+ type IClickhouseSession,+ type IServiceCreateEventPayload,+ sessionBuffer,+} from '@openpanel/db';+import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';+import type { Job } from 'bullmq';+import { beforeEach, describe, expect, it, vi } from 'vitest';+import { createSessionEnd } from './events.create-session-end';++const { redisMock } = vi.hoisted(() => ({+ redisMock: { set: vi.fn() },+}));++vi.mock('@openpanel/redis', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@openpanel/redis')>();+ return { ...actual, getRedisCache: () => redisMock };+});++vi.mock('@openpanel/db', async (importOriginal) => {+ const actual = await importOriginal<typeof import('@openpanel/db')>();+ return {+ ...actual,+ createEvent: vi.fn().mockResolvedValue({ document: { id: 'evt-1' } }),+ getNotificationRulesByProjectId: vi.fn().mockResolvedValue([]),+ getHasFunnelRules: vi.fn().mockReturnValue(false),+ transformEvent: vi.fn((e) => e),+ sessionBuffer: { getExistingSession: vi.fn(), cleanup: vi.fn() },+ };+});++vi.mock('@/metrics', () => {+ const m = () => ({ inc: vi.fn(), observe: vi.fn(), set: vi.fn() });+ return {+ sessionDurationOnClose: m(),+ sessionEndsEmitted: m(),+ sessionEndsSkipped: m(),+ sessionEventsOnClose: m(),+ };+});++const getExistingSession = vi.mocked(sessionBuffer.getExistingSession);+const cleanup = vi.mocked(sessionBuffer.cleanup);+const createEventMock = vi.mocked(createEvent);++const snapshot: IClickhouseSession = {+ id: 'sess-1',+ project_id: 'proj-1',+ device_id: 'dev-1',+ profile_id: 'dev-1',+ created_at: '2026-06-08 10:30:00',+ ended_at: '2026-06-08 11:00:00',+ is_bounce: false,+ duration: 1_800_000,+ event_count: 3,+ screen_view_count: 5,+ exit_path: '/end',+} as unknown as IClickhouseSession;++const makeJob = (+ snap: IClickhouseSession+): Job<EventsQueuePayloadCreateSessionEnd> =>+ ({+ id: 'job-1',+ data: {+ type: 'createSessionEnd',+ payload: {+ projectId: 'proj-1',+ deviceId: 'dev-1',+ profileId: 'dev-1',+ name: 'session_end',+ properties: {},+ } as unknown as IServiceCreateEventPayload,+ snapshot: snap,+ },+ }) as unknown as Job<EventsQueuePayloadCreateSessionEnd>;++beforeEach(() => {+ vi.clearAllMocks();+ redisMock.set.mockResolvedValue('OK'); // claim acquired by default+ cleanup.mockResolvedValue(undefined as never);+ createEventMock.mockResolvedValue({ document: { id: 'evt-1' } } as never);+});++describe('createSessionEnd', () => {+ it('emits session_end and id-gated cleanup on the happy path', async () => {+ getExistingSession.mockResolvedValue(snapshot); // live == snapshot, unchanged++ await createSessionEnd(makeJob(snapshot));++ expect(createEventMock).toHaveBeenCalledTimes(1);+ expect(createEventMock.mock.calls[0]![0]).toMatchObject({+ name: 'session_end',+ sessionId: 'sess-1',+ });+ expect(cleanup).toHaveBeenCalledWith({+ projectId: 'proj-1',+ deviceId: 'dev-1',+ sessionId: 'sess-1',+ profileId: 'dev-1',+ });+ });++ it('skips when the session was extended after the close was enqueued', async () => {+ getExistingSession.mockResolvedValue({+ ...snapshot,+ ended_at: '2026-06-08 11:05:00', // later than snapshot → still active+ });++ const result = await createSessionEnd(makeJob(snapshot));++ expect(result).toBeNull();+ expect(createEventMock).not.toHaveBeenCalled();+ expect(cleanup).not.toHaveBeenCalled();+ });++ it('skips when the idempotency claim is already taken', async () => {+ getExistingSession.mockResolvedValue(snapshot);+ redisMock.set.mockResolvedValue(null); // SET NX failed → already emitted++ const result = await createSessionEnd(makeJob(snapshot));++ expect(result).toBeNull();+ expect(createEventMock).not.toHaveBeenCalled();+ });++ it('uses the snapshot when a new session already owns the slot (boundary)', async () => {+ // A boundary opened a fresh session under the same (project, device) slot.+ getExistingSession.mockResolvedValue({+ ...snapshot,+ id: 'sess-2',+ ended_at: '2026-06-08 12:00:00',+ });++ await createSessionEnd(makeJob(snapshot));++ // Emits for the CLOSED session (snapshot), not the live one.+ expect(createEventMock.mock.calls[0]![0]).toMatchObject({+ sessionId: 'sess-1',+ });+ // cleanup is keyed on the closed id → id-gated Lua no-ops against sess-2.+ expect(cleanup).toHaveBeenCalledWith(+ expect.objectContaining({ sessionId: 'sess-1' })+ );+ });+});diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex 32e901ad6..460a40d76 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -16,45 +16,40 @@ import {transformSessionToEvent,} from '@openpanel/db';import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';+import { getRedisCache } from '@openpanel/redis';import type { Job } from 'bullmq';import { logger as baseLogger } from '@/utils/logger';+import {+ sessionDurationOnClose,+ sessionEndsEmitted,+ sessionEndsSkipped,+ sessionEventsOnClose,+} from '@/metrics';const MAX_SESSION_EVENTS = 500;+const SESSION_END_CLAIM_TTL = 60 * 60 * 2; // 2h, well past any plausible retry window-// Grabs session_start and the last occured eventasync function getSessionEvents({- projectId,- sessionId,+ session,startAt,endAt,}: {- projectId: string;- sessionId: string;+ session: IClickhouseSession;startAt: Date;endAt: Date;}): Promise<IServiceEvent[]> {const sql = `SELECT * FROM ${TABLE_NAMES.events}WHERE- session_id = '${sessionId}'- AND project_id = '${projectId}'+ session_id = '${session.id}'+ AND project_id = '${session.project_id}'AND created_at BETWEEN '${formatClickhouseDate(new Date(startAt.getTime() - 1000))}' AND '${formatClickhouseDate(new Date(endAt.getTime() + 1000))}'ORDER BY created_at DESC LIMIT ${MAX_SESSION_EVENTS};`;- const [lastScreenView, eventsInDb] = await Promise.all([- sessionBuffer.getExistingSession({- sessionId,- }),- getEvents(sql),- ]);-- // sort last inserted first- return [- lastScreenView ? transformSessionToEvent(lastScreenView) : null,- ...eventsInDb,- ]- .flatMap((event) => (event ? [event] : []))+ const eventsInDb = await getEvents(sql);++ return [transformSessionToEvent(session), ...eventsInDb].sort((a, b) =>new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()@@ -64,7 +59,7 @@ async function getSessionEvents({export async function createSessionEnd(job: Job<EventsQueuePayloadCreateSessionEnd>) {- const { payload } = job.data;+ const { payload, snapshot } = job.data;const logger = baseLogger.child({payload,jobId: job.id,@@ -72,14 +67,65 @@ export async function createSessionEnd(logger.debug('Processing session end job');- const session = await sessionBuffer.getExistingSession({- sessionId: payload.sessionId,+ // Prefer the live blob — it reflects any late extensions that arrived after+ // enqueue. Fall back to the snapshot if the blob expired (very long queue+ // lag, retries hours later, etc.).+ const live = await sessionBuffer.getExistingSession({+ projectId: payload.projectId,+ deviceId: payload.deviceId,});+ // Distinguish three cases:+ // - Same session, extended after enqueue → bail. Reaper will retry later.+ // - Same session, no change → use live (snapshot equivalent).+ // - Different session in slot (boundary) → use snapshot for emission;+ // cleanup will be a no-op since the slot is owned by the new session.+ // - No live blob at all (TTL expired) → use snapshot.+ const sameSession = live && live.id === snapshot.id;+ if (sameSession && live.ended_at > snapshot.ended_at) {+ sessionEndsSkipped.inc({ reason: 'extended_after_enqueue' });+ logger.info(+ {+ sessionId: live.id,+ projectId: live.project_id,+ snapshotEndedAt: snapshot.ended_at,+ liveEndedAt: live.ended_at,+ },+ 'session was extended after close was enqueued, skipping',+ );+ return null;+ }++ const session = sameSession ? live : snapshot;if (!session) {- throw new Error('Session not found');+ sessionEndsSkipped.inc({ reason: 'not_found' });+ logger.warn(+ { projectId: payload.projectId, deviceId: payload.deviceId },+ 'No live session and no snapshot — skipping session_end',+ );+ return null;}+ // Idempotency claim: first writer wins. Subsequent retries / reaper+ // double-fires are no-ops.+ const redis = getRedisCache();+ const claimKey = `session:end:emitted:${session.project_id}:${session.device_id}:${session.id}`;+ const claimed = await redis.set(claimKey, '1', 'EX', SESSION_END_CLAIM_TTL, 'NX');+ if (claimed === null) {+ sessionEndsSkipped.inc({ reason: 'already_emitted' });+ logger.info(+ { sessionId: session.id, projectId: session.project_id },+ 'session_end already emitted, skipping',+ );+ return null;+ }++ sessionEndsEmitted.inc();+ sessionDurationOnClose.observe(Math.max(0, session.duration ?? 0));+ sessionEventsOnClose.observe(+ (session.event_count ?? 0) + (session.screen_view_count ?? 0)+ );+const profileId = session.profile_id || payload.profileId;if (@@ -96,7 +142,7 @@ export async function createSessionEnd() {await profileBackfillBuffer.add({projectId: payload.projectId,- sessionId: payload.sessionId,+ sessionId: session.id,profileId,});}@@ -105,6 +151,7 @@ export async function createSessionEnd(// Create session end eventconst { document: sessionEndEvent } = await createEvent({...payload,+ sessionId: session.id,properties: {...payload.properties,__bounce: session.is_bounce,@@ -131,6 +178,23 @@ export async function createSessionEnd();}+ // Clean up Redis state for this session. cleanup() is id-gated — if a new+ // session has already taken over this (projectId, deviceId) slot via a+ // boundary, the call is a no-op and the new session is preserved.+ await sessionBuffer+ .cleanup({+ projectId: session.project_id,+ deviceId: session.device_id,+ sessionId: session.id,+ profileId: session.profile_id,+ })+ .catch((error) => {+ logger.error(+ { err: error, sessionId: session.id },+ 'Failed to cleanup session state after session_end',+ );+ });+return sessionEndEvent;}@@ -152,8 +216,7 @@ async function handleSessionEndNotifications({if (hasFunnelRules && isEventCountReasonable) {const events = await getSessionEvents({- projectId: payload.projectId,- sessionId: payload.sessionId,+ session,startAt: convertClickhouseDateToJs(session.created_at),endAt: convertClickhouseDateToJs(session.ended_at),});diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex cf89391fe..0cac12864 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -11,22 +11,16 @@ import {import type { ILogger } from '@openpanel/logger';import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';+import { sessionEndsEnqueued, sessionsStarted } from '@/metrics';import { logger as baseLogger } from '@/utils/logger';-import {- createSessionEndJob,- extendSessionEndJob,- getActiveSessionEndJob,-} from '@/utils/session-handler';+import { enqueueSessionEndV2 } from '@/utils/session-handler';const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];-// This function will merge two objects.-// First it will strip '' and undefined/null from B-// Then it will merge the two objects with a standard ramda merge function+// Strip empty/nullish from B, then deep-merge over A.const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>mergeDeepRight(a, reject(anyPass([isEmpty, isNil]))(b)) as A & B;-/** Check if payload matches project-level event exclude filters */async function isEventExcludedByProjectFilter(payload: IServiceCreateEventPayload,projectId: string@@ -46,12 +40,11 @@ async function createEventAndNotify(logger: ILogger,projectId: string) {- // Check project-level event exclude filtersconst isExcluded = await isEventExcludedByProjectFilter(payload, projectId);if (isExcluded) {logger.info({ event: payload.name, projectId },- 'Event excluded by project filter',+ 'Event excluded by project filter');return null;}@@ -61,7 +54,6 @@ async function createEventAndNotify(createEvent(payload),checkNotificationRulesForEvent(payload).catch(() => null),]);-return event;}@@ -74,10 +66,7 @@ const parseRevenue = (revenue: unknown): number | undefined => {}if (typeof revenue === 'string') {const parsed = Number.parseFloat(revenue);- if (Number.isNaN(parsed)) {- return undefined;- }- return parsed;+ return Number.isNaN(parsed) ? undefined : parsed;}return undefined;};@@ -117,7 +106,6 @@ export async function incomingEvent();};- // this will get the profileId from the alias table if it existsconst profileId = body.profileId ? String(body.profileId) : '';const createdAt = new Date(body.timestamp);const isTimestampFromThePast = body.isTimestampFromThePast;@@ -169,14 +157,13 @@ export async function incomingEvent(: undefined,};- // if timestamp is from the past we dont want to create a new session+ // Server-side and "timestamp from the past" events ride alongside an+ // existing client session if there is one — they don't open / close+ // sessions of their own.if (uaInfo.isServer || isTimestampFromThePast) {const session =profileId && !isTimestampFromThePast- ? await sessionBuffer.getExistingSession({- profileId,- projectId,- })+ ? await sessionBuffer.getExistingSession({ profileId, projectId }): null;const payload = {@@ -205,60 +192,67 @@ export async function incomingEvent(return createEventAndNotify(payload as IServiceEvent, logger, projectId);}- const activeSessionEndJob = await getActiveSessionEndJob(- projectId,- deviceId,- );- const activeSessionPayload = activeSessionEndJob?.data.payload;-- const payload: IServiceCreateEventPayload = merge(baseEvent, {- referrer: activeSessionPayload?.referrer ?? baseEvent.referrer,- referrerName: activeSessionPayload?.referrerName ?? baseEvent.referrerName,- referrerType: activeSessionPayload?.referrerType ?? baseEvent.referrerType,- } as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;-- const isExcluded = await isEventExcludedByProjectFilter(payload, projectId);- if (isExcluded) {+ if (await isEventExcludedByProjectFilter(baseEvent, projectId)) {logger.info(- { event: payload.name, projectId },- 'Skipping session_start and event (excluded by project filter)',+ { event: baseEvent.name, projectId },+ 'Skipping session_start and event (excluded by project filter)');return null;}- if (activeSessionEndJob) {- await extendSessionEndJob({- projectId,- deviceId,- job: activeSessionEndJob,- }).catch((error) => {- logger.warn({ err: error }, 'Failed to extend session end job');- });- } else {+ // The single source of truth for session lifecycle. Reads the current+ // session, decides extend/new/boundary, writes back. The returned+ // `current` is the canonical session — use its referrer fields for+ // inheritance, just like the previous behavior.+ const session = await sessionBuffer.ingest(baseEvent);++ if (session?.kind === 'boundary') {+ // Close the old session in a separate job (one Redis-buffered insert+ // for the session_end event + notification rule check). Idempotent via+ // BullMQ jobId dedup.+ await enqueueSessionEndV2({+ payload: baseEvent,+ closedSession: session.closed,+ })+ .then(() => sessionEndsEnqueued.inc({ source: 'boundary' }))+ .catch((error) => {+ logger.error(+ { err: error, deviceId, sessionId: session.closed.id },+ 'Error enqueueing session_end on boundary'+ );+ });+ }++ if (session?.kind === 'new' || session?.kind === 'boundary') {+ sessionsStarted.inc({ kind: session.kind });await createEventAndNotify({- ...payload,+ ...baseEvent,name: 'session_start',- createdAt: new Date(getTime(payload.createdAt) - 100),+ createdAt: new Date(getTime(baseEvent.createdAt) - 100),},logger,projectId).catch((error) => {logger.error(- { err: error, event: payload },- 'Error creating session start event',- );- throw error;- });-- await createSessionEndJob({ payload }).catch((error) => {- logger.error(- { err: error, event: payload },- 'Error creating session end job',+ { err: error, event: baseEvent },+ 'Error creating session start event');throw error;});}- return createEventAndNotify(payload, logger, projectId);+ // Inherit referrer fields from the canonical session for the actual event.+ // For 'extend' this preserves the original session's referrer across+ // mid-session events; for 'new' / 'boundary' it's the event's own referrer+ // (which `ingest` just stored on the fresh session).+ const finalPayload: IServiceCreateEventPayload = session+ ? merge(baseEvent, {+ referrer: session.current.referrer,+ referrerName: session.current.referrer_name,+ referrerType: session.current.referrer_type,+ } as Partial<IServiceCreateEventPayload>)+ : baseEvent;++ return createEventAndNotify(finalPayload, logger, projectId);}diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.tsindex 30547d7dc..abb22e080 100644--- a/apps/worker/src/jobs/events.incoming-events.test.ts+++ b/apps/worker/src/jobs/events.incoming-events.test.ts@@ -22,16 +22,18 @@ vi.mock('@openpanel/db', async () => {getProjectByIdCached: vi.fn().mockResolvedValue({ filters: [] }),matchEvent: vi.fn().mockReturnValue(false),sessionBuffer: {+ // name/getBufferSize keep metrics.ts happy (it registers a per-buffer+ // gauge at import); the tests only drive getExistingSession/ingest.+ name: 'session',+ getBufferSize: vi.fn().mockResolvedValue(0),getExistingSession: vi.fn(),+ ingest: vi.fn(),},};});-// 30 minutes-const SESSION_TIMEOUT = 30 * 60 * 1000;const projectId = 'test-project';const deviceId = 'device-123';-// Valid UUID used when creating a new session in testsconst newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';const geo = {country: 'US',@@ -63,194 +65,164 @@ const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {model: '',};+function makeSession(+ overrides: Partial<IClickhouseSession> = {}+): IClickhouseSession {+ const now = new Date();+ return {+ id: 'session-existing',+ project_id: projectId,+ device_id: deviceId,+ profile_id: '',+ event_count: 1,+ screen_view_count: 0,+ entry_path: '/',+ entry_origin: 'https://example.com',+ exit_path: '/',+ exit_origin: 'https://example.com',+ created_at: formatClickhouseDate(now),+ ended_at: formatClickhouseDate(now),+ os: 'Windows',+ os_version: '10',+ browser: 'Chrome',+ browser_version: '91.0.4472.124',+ device: 'desktop',+ brand: '',+ model: '',+ country: 'US',+ region: 'NY',+ city: 'New York',+ longitude: 0,+ latitude: 0,+ duration: 0,+ referrer: '',+ referrer_name: '',+ referrer_type: '',+ is_bounce: true,+ utm_term: '',+ utm_source: '',+ utm_campaign: '',+ utm_content: '',+ utm_medium: '',+ revenue: 0,+ sign: 1,+ version: 1,+ groups: [],+ ...overrides,+ } satisfies IClickhouseSession;+}++function buildJobData(+ overrides: Partial<EventsQueuePayloadIncomingEvent['payload']> = {}+): EventsQueuePayloadIncomingEvent['payload'] {+ return {+ geo,+ event: {+ name: 'test_event',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ properties: { __path: 'https://example.com/test' },+ },+ uaInfo,+ headers: {+ 'request-id': '123',+ 'user-agent': 'Mozilla/5.0',+ 'openpanel-sdk-name': 'web',+ 'openpanel-sdk-version': '1.0.0',+ },+ projectId,+ deviceId,+ sessionId: newSessionId,+ ...overrides,+ };+}+describe('incomingEvent', () => {beforeEach(() => {vi.clearAllMocks();+ (createEvent as Mock).mockImplementation((event) => event);});- it('should create a session start and an event', async () => {- const spySessionsQueueAdd = vi- .spyOn(sessionsQueue, 'add')- .mockResolvedValue({} as Job);- const timestamp = new Date();- // Mock job data- const jobData: EventsQueuePayloadIncomingEvent['payload'] = {- geo,- event: {- name: 'test_event',- timestamp: timestamp.toISOString(),- isTimestampFromThePast: false,- properties: { __path: 'https://example.com/test' },- },- uaInfo,- headers: {- 'request-id': '123',- 'user-agent':- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',- 'openpanel-sdk-name': 'web',- 'openpanel-sdk-version': '1.0.0',- },- projectId,- deviceId,- sessionId: newSessionId,- };- const event = {- name: 'test_event',- deviceId,- profileId: '',- sessionId: expect.stringMatching(- // biome-ignore lint/performance/useTopLevelRegex: test- /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i- ),- projectId,- properties: {- __hash: undefined,- __query: undefined,- },- createdAt: timestamp,- country: 'US',- city: 'New York',- region: 'NY',- revenue: undefined,- longitude: 0,- latitude: 0,- os: 'Windows',- osVersion: '10',- browser: 'Chrome',- browserVersion: '91.0.4472.124',- device: 'desktop',- brand: '',- model: '',- duration: 0,- path: '/test',- origin: 'https://example.com',- referrer: '',- referrerName: '',- referrerType: '',- sdkName: jobData.headers['openpanel-sdk-name'],- sdkVersion: jobData.headers['openpanel-sdk-version'],- groups: [],- };+ it('emits session_start when ingest returns kind="new"', async () => {+ vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({+ kind: 'new',+ current: makeSession({ id: newSessionId }),+ });- (createEvent as Mock).mockReturnValue(event);+ await incomingEvent(buildJobData());- // Execute the job- await incomingEvent(jobData);+ expect(sessionsQueue.add).not.toHaveBeenCalled();+ const calls = (createEvent as Mock).mock.calls;+ expect(calls).toHaveLength(2);+ expect(calls[0]![0].name).toBe('session_start');+ expect(calls[1]![0].name).toBe('test_event');+ });- expect(spySessionsQueueAdd).toHaveBeenCalledWith(- 'session',- {- type: 'createSessionEnd',- payload: expect.objectContaining(event),- },- {- delay: SESSION_TIMEOUT,- jobId: `sessionEnd:${projectId}:${deviceId}`,- attempts: 3,- backoff: {- delay: 200,- type: 'exponential',- },- }+ it('skips session_start when ingest returns kind="extend"', async () => {+ vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({+ kind: 'extend',+ current: makeSession(),+ });++ await incomingEvent(buildJobData());++ expect(sessionsQueue.add).not.toHaveBeenCalled();+ const sessionStartCalls = (createEvent as Mock).mock.calls.filter(+ ([arg]) => arg?.name === 'session_start');+ expect(sessionStartCalls).toHaveLength(0);+ });- expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({- ...event,- createdAt: new Date(timestamp.getTime() - 100),- name: 'session_start',+ it('closes old session and emits session_start when ingest returns kind="boundary"', async () => {+ const closed = makeSession({ id: 'old-session-id' });+ const current = makeSession({ id: newSessionId });+ vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({+ kind: 'boundary',+ current,+ closed,});- expect((createEvent as Mock).mock.calls[1]).toMatchObject([event]);- });- it('should reuse existing session', async () => {- const spySessionsQueueAdd = vi.spyOn(sessionsQueue, 'add');- const spySessionsQueueGetJob = vi.spyOn(sessionsQueue, 'getJob');+ const spy = vi+ .spyOn(sessionsQueue, 'add')+ .mockResolvedValue({} as Job);- const timestamp = new Date();- // Mock job data- const jobData: EventsQueuePayloadIncomingEvent['payload'] = {- geo,- event: {- name: 'test_event',- timestamp: timestamp.toISOString(),- properties: { __path: 'https://example.com/test' },- isTimestampFromThePast: false,- },- headers: {- 'request-id': '123',- 'user-agent':- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',- 'openpanel-sdk-name': 'web',- 'openpanel-sdk-version': '1.0.0',- },- uaInfo,- projectId,- deviceId,- sessionId: 'session-123',- };+ await incomingEvent(buildJobData());- const changeDelay = vi.fn();- const updateData = vi.fn();- spySessionsQueueGetJob.mockResolvedValueOnce({- getState: vi.fn().mockResolvedValue('delayed'),- updateData,- changeDelay,- data: {- type: 'createSessionEnd',- payload: {- sessionId: 'session-123',- deviceId,- },- },- } as Partial<Job> as Job);- // Execute the job- await incomingEvent(jobData);+ expect(spy).toHaveBeenCalledTimes(1);+ const [, payload, opts] = spy.mock.calls[0]!;+ expect((payload as any).type).toBe('createSessionEnd');+ expect((payload as any).payload.sessionId).toBe('old-session-id');+ expect((payload as any).snapshot.id).toBe('old-session-id');+ expect(opts?.jobId).toBe('sessionEnd:v2:old-session-id');- const event = {- name: 'test_event',- deviceId,- profileId: '',- sessionId: 'session-123',- projectId,- properties: {- __hash: undefined,- __query: undefined,- },- createdAt: timestamp,- country: 'US',- city: 'New York',- region: 'NY',- revenue: undefined,- longitude: 0,- latitude: 0,- os: 'Windows',- osVersion: '10',- browser: 'Chrome',- browserVersion: '91.0.4472.124',- device: 'desktop',- brand: '',- model: '',- duration: 0,- path: '/test',- origin: 'https://example.com',- referrer: '',- referrerName: '',- referrerType: '',- sdkName: jobData.headers['openpanel-sdk-name'],- sdkVersion: jobData.headers['openpanel-sdk-version'],- groups: [],- };+ const calls = (createEvent as Mock).mock.calls;+ expect(calls.filter(([a]) => a?.name === 'session_start')).toHaveLength(1);+ expect(calls.filter(([a]) => a?.name === 'test_event')).toHaveLength(1);+ });++ it('inherits referrer from current session on the actual event', async () => {+ vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({+ kind: 'extend',+ current: makeSession({+ referrer: 'https://google.com',+ referrer_name: 'Google',+ referrer_type: 'search',+ }),+ });++ await incomingEvent(buildJobData());- expect(spySessionsQueueAdd).toHaveBeenCalledTimes(0);- expect(changeDelay).toHaveBeenCalledWith(SESSION_TIMEOUT);- expect(createEvent as Mock).toBeCalledTimes(1);- expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual(event);+ const testEventCall = (createEvent as Mock).mock.calls.find(+ ([a]) => a?.name === 'test_event'+ );+ expect(testEventCall![0].referrer).toBe('https://google.com');+ expect(testEventCall![0].referrerName).toBe('Google');+ expect(testEventCall![0].referrerType).toBe('search');});- it('should handle server events (with existing screen view)', async () => {+ it('handles server events with existing profile session', async () => {const timestamp = new Date();- const jobData: EventsQueuePayloadIncomingEvent['payload'] = {- geo,+ const jobData = buildJobData({event: {name: 'server_event',timestamp: timestamp.toISOString(),@@ -258,231 +230,109 @@ describe('incomingEvent', () => {profileId: 'profile-123',isTimestampFromThePast: false,},+ uaInfo: uaInfoServer,+ deviceId: '',+ sessionId: '',headers: {'user-agent': 'OpenPanel Server/1.0','openpanel-sdk-name': 'server','openpanel-sdk-version': '1.0.0','request-id': '123',},- projectId,- deviceId: '',- sessionId: '',- uaInfo: uaInfoServer,- };+ });- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce({- id: 'last-session-456',- event_count: 0,- screen_view_count: 0,- entry_path: '/last-path',- entry_origin: 'https://example.com',- exit_path: '/last-path',- exit_origin: 'https://example.com',- created_at: formatClickhouseDate(timestamp),- ended_at: formatClickhouseDate(timestamp),- os: 'iOS',- os_version: '15.0',- browser: 'Safari',- browser_version: '15.0',- device: 'mobile',- brand: 'Apple',- model: 'iPhone',- country: 'CA',- region: 'ON',- city: 'Toronto',- longitude: 0,- latitude: 0,- duration: 0,- referrer: 'https://google.com',- referrer_name: 'Google',- referrer_type: 'search',- is_bounce: false,- utm_term: '',- utm_source: '',- utm_campaign: '',- utm_content: '',- utm_medium: '',- revenue: 0,- project_id: projectId,- device_id: 'last-device-123',- profile_id: 'profile-123',- sign: 1,- version: 1,- groups: [],- } satisfies IClickhouseSession);+ vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(+ makeSession({+ id: 'last-session-456',+ device_id: 'last-device-123',+ profile_id: 'profile-123',+ os: 'iOS',+ os_version: '15.0',+ browser: 'Safari',+ browser_version: '15.0',+ device: 'mobile',+ brand: 'Apple',+ model: 'iPhone',+ country: 'CA',+ region: 'ON',+ city: 'Toronto',+ entry_path: '/last-path',+ entry_origin: 'https://example.com',+ exit_path: '/last-path',+ exit_origin: 'https://example.com',+ referrer: 'https://google.com',+ referrer_name: 'Google',+ referrer_type: 'search',+ is_bounce: false,+ event_count: 0,+ screen_view_count: 0,+ })+ );await incomingEvent(jobData);- expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({+ expect(sessionBuffer.ingest).not.toHaveBeenCalled();+ expect(sessionsQueue.add).not.toHaveBeenCalled();+ expect((createEvent as Mock).mock.calls[0]![0]).toMatchObject({name: 'server_event',deviceId: 'last-device-123',sessionId: 'last-session-456',profileId: 'profile-123',- projectId,- properties: {- custom_property: 'test_value',- __hash: undefined,- __query: undefined,- },- createdAt: timestamp,- country: 'CA',city: 'Toronto',- region: 'ON',- longitude: 0,- latitude: 0,- os: 'iOS',- osVersion: '15.0',- browser: 'Safari',- browserVersion: '15.0',- device: 'mobile',- brand: 'Apple',- model: 'iPhone',- duration: 0,- path: '/last-path',- origin: 'https://example.com',+ country: 'CA',referrer: 'https://google.com',- referrerName: 'Google',- referrerType: 'search',- sdkName: 'server',- sdkVersion: '1.0.0',- revenue: undefined,- groups: [],});-- expect(sessionsQueue.add).not.toHaveBeenCalled();});- it('should handle server events (without existing screen view)', async () => {- const timestamp = new Date();- const jobData: EventsQueuePayloadIncomingEvent['payload'] = {- geo,- event: {- name: 'server_event',- timestamp: timestamp.toISOString(),- properties: { custom_property: 'test_value' },- profileId: 'profile-123',- isTimestampFromThePast: false,- },- headers: {- 'user-agent': 'OpenPanel Server/1.0',- 'openpanel-sdk-name': 'server',- 'openpanel-sdk-version': '1.0.0',- 'request-id': '123',- },- projectId,- deviceId: '',- sessionId: '',- uaInfo: uaInfoServer,- };+ it('handles server events without any active session', async () => {+ vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);- await incomingEvent(jobData);+ await incomingEvent(+ buildJobData({+ event: {+ name: 'server_event',+ timestamp: new Date().toISOString(),+ properties: { custom_property: 'test_value' },+ profileId: 'profile-123',+ isTimestampFromThePast: false,+ },+ uaInfo: uaInfoServer,+ deviceId: '',+ sessionId: '',+ headers: {+ 'user-agent': 'OpenPanel Server/1.0',+ 'openpanel-sdk-name': 'server',+ 'openpanel-sdk-version': '1.0.0',+ 'request-id': '123',+ },+ })+ );- expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({+ expect(sessionBuffer.ingest).not.toHaveBeenCalled();+ expect(sessionsQueue.add).not.toHaveBeenCalled();+ expect((createEvent as Mock).mock.calls[0]![0]).toMatchObject({name: 'server_event',deviceId: '',sessionId: '',profileId: 'profile-123',- projectId,- properties: {- custom_property: 'test_value',- __hash: undefined,- __query: undefined,- },- createdAt: timestamp,- country: 'US',- city: 'New York',- region: 'NY',- revenue: undefined,- longitude: 0,- latitude: 0,- os: '',- osVersion: '',- browser: '',- browserVersion: '',- device: 'server',- brand: '',- model: '',- duration: 0,- path: '',- origin: '',- referrer: undefined,- referrerName: undefined,- referrerType: undefined,- sdkName: 'server',- sdkVersion: '1.0.0',- groups: [],});-- expect(sessionsQueue.add).not.toHaveBeenCalled();});- it('should emit session_start only once when 3 events arrive in rapid succession', async () => {- // Regression test: previously the API baked `session: undefined` into every- // payload when no session-end job existed yet. Even with sequential- // per-device processing in the worker, the worker re-checks the BullMQ- // session-end job at processing time, so events 2 and 3 should extend- // rather than emit duplicate session_starts.- const spySessionsQueueAdd = vi- .spyOn(sessionsQueue, 'add')- .mockResolvedValue({} as Job);- const spySessionsQueueGetJob = vi.spyOn(sessionsQueue, 'getJob');+ it('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {+ const session = makeSession({ id: newSessionId });+ vi.mocked(sessionBuffer.ingest)+ .mockResolvedValueOnce({ kind: 'new', current: session })+ .mockResolvedValueOnce({ kind: 'extend', current: session })+ .mockResolvedValueOnce({ kind: 'extend', current: session });- const buildJobData = (- eventName: string,- ): EventsQueuePayloadIncomingEvent['payload'] => ({- geo,- event: {- name: eventName,- timestamp: new Date().toISOString(),- isTimestampFromThePast: false,- properties: { __path: 'https://example.com/test' },- },- uaInfo,- headers: {- 'request-id': '123',- 'user-agent':- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',- 'openpanel-sdk-name': 'web',- 'openpanel-sdk-version': '1.0.0',- },- projectId,- deviceId,- sessionId: newSessionId,- });-- // Event 1: no session-end job exists yet → emit session_start.- spySessionsQueueGetJob.mockResolvedValueOnce(undefined);- // Events 2 and 3: session-end job is now present (delayed) → extend only.- const liveJob = {- id: `sessionEnd:${projectId}:${deviceId}`,- getState: vi.fn().mockResolvedValue('delayed'),- changeDelay: vi.fn(),- data: {- type: 'createSessionEnd',- payload: {- sessionId: newSessionId,- deviceId,- referrer: '',- referrerName: '',- referrerType: '',- },- },- } as Partial<Job> as Job;- spySessionsQueueGetJob.mockResolvedValue(liveJob);-- (createEvent as Mock).mockImplementation((event) => event);-- await incomingEvent(buildJobData('event_a'));- await incomingEvent(buildJobData('event_b'));- await incomingEvent(buildJobData('event_c'));+ await incomingEvent(buildJobData({ event: { name: 'e1', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));+ await incomingEvent(buildJobData({ event: { name: 'e2', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));+ await incomingEvent(buildJobData({ event: { name: 'e3', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));const sessionStartCalls = (createEvent as Mock).mock.calls.filter(- ([arg]) => arg?.name === 'session_start',+ ([a]) => a?.name === 'session_start');expect(sessionStartCalls).toHaveLength(1);-- // Only the first event should have queued a session-end job; subsequent- // events extend the existing one via changeDelay.- expect(spySessionsQueueAdd).toHaveBeenCalledTimes(1);+ expect(sessionsQueue.add).not.toHaveBeenCalled();});});diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex a197d9865..2f595e8a0 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -8,6 +8,7 @@ import {sessionBuffer,} from '@openpanel/db';import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';+import { getRedisCache } from '@openpanel/redis';import client from 'prom-client';const Registry = client.Registry;@@ -124,7 +125,7 @@ for (const buf of allBuffers) {// ignore — scrape continues}},- }),+ }));}@@ -141,7 +142,7 @@ register.registerMetric(// ignore}},- }),+ }));// ---- Flush metrics (populated via flushObserver hooks) ----@@ -150,7 +151,9 @@ const flushDuration = new client.Histogram({name: 'buffer_flush_duration_ms',help: 'Wall time of a tryFlush call, including lock acquisition',labelNames: ['buffer', 'result', 'trigger'],- buckets: [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000],+ buckets: [+ 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,+ ],});register.registerMetric(flushDuration);@@ -181,7 +184,9 @@ const chInsertDurationMs = new client.Histogram({name: 'buffer_ch_insert_duration_ms',help: 'Duration of the ClickHouse insert(s) inside a single flush',labelNames: ['buffer'],- buckets: [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000],+ buckets: [+ 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,+ ],});register.registerMetric(chInsertDurationMs);@@ -193,7 +198,9 @@ const chFetchDurationMs = new client.Histogram({name: 'buffer_ch_fetch_duration_ms',help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',labelNames: ['buffer'],- buckets: [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000],+ buckets: [+ 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,+ ],});register.registerMetric(chFetchDurationMs);@@ -201,7 +208,10 @@ const flushLlenAtStart = new client.Histogram({name: 'buffer_flush_llen_at_start',help: 'LLEN observed at the start of a flush attempt',labelNames: ['buffer'],- buckets: [0, 10, 100, 500, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000],+ buckets: [+ 0, 10, 100, 500, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000,+ 5_000_000,+ ],});register.registerMetric(flushLlenAtStart);@@ -238,7 +248,7 @@ for (const buf of allBuffers) {});flushDuration.observe({ buffer: obs.buffer, result: obs.result, trigger: obs.trigger },- obs.totalMs,+ obs.totalMs);if (obs.llenAtStart != null) {@@ -252,26 +262,20 @@ for (const buf of allBuffers) {if (obs.phases?.lrangeMs != null) {redisOpDurationMs.observe({ buffer: obs.buffer, op: 'lrange' },- obs.phases.lrangeMs,+ obs.phases.lrangeMs);}if (obs.phases?.trimMs != null) {redisOpDurationMs.observe({ buffer: obs.buffer, op: 'trim' },- obs.phases.trimMs,+ obs.phases.trimMs);}if (obs.phases?.chFetchMs != null) {- chFetchDurationMs.observe(- { buffer: obs.buffer },- obs.phases.chFetchMs,- );+ chFetchDurationMs.observe({ buffer: obs.buffer }, obs.phases.chFetchMs);}if (obs.phases?.chInsertMs != null) {- chInsertDurationMs.observe(- { buffer: obs.buffer },- obs.phases.chInsertMs,- );+ chInsertDurationMs.observe({ buffer: obs.buffer }, obs.phases.chInsertMs);}};@@ -288,3 +292,159 @@ for (const buf of allBuffers) {addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);};}++// Note: `buffer_replay_count` is already registered by the allBuffers loop+// above (replayBuffer is in that list), so no standalone gauge here — a second+// registration with the same name throws and crashes the worker on boot.++// -----------------------------------------------------------------------+// Session lifecycle metrics (new session-buffer + reaper world)+// -----------------------------------------------------------------------++// Counters incremented at runtime by the session lifecycle code paths.++export const sessionsStarted = new client.Counter({+ name: 'sessions_started_total',+ help: 'session_start events emitted, by lifecycle kind',+ labelNames: ['kind'], // 'new' | 'boundary'+});+register.registerMetric(sessionsStarted);++export const sessionEndsEnqueued = new client.Counter({+ name: 'session_ends_enqueued_total',+ help: 'session_end jobs pushed onto the sessions queue, by trigger source',+ labelNames: ['source'], // 'boundary' | 'reaper'+});+register.registerMetric(sessionEndsEnqueued);++export const sessionEndsEmitted = new client.Counter({+ name: 'session_ends_emitted_total',+ help: 'session_end events actually written (post-idempotency claim)',+});+register.registerMetric(sessionEndsEmitted);++export const sessionEndsSkipped = new client.Counter({+ name: 'session_ends_skipped_total',+ help: 'session_end jobs that ran but did not emit, by reason',+ labelNames: ['reason'], // 'not_found' | 'already_emitted'+});+register.registerMetric(sessionEndsSkipped);++export const sessionsReaped = new client.Counter({+ name: 'sessions_reaped_total',+ help: 'Sessions closed by the reaper, by trigger condition',+ labelNames: ['reason'], // 'event-time' | 'deadman'+});+register.registerMetric(sessionsReaped);++export const sessionsReaperOrphans = new client.Counter({+ name: 'sessions_reaper_orphans_total',+ help: 'Reaper found a sorted-set entry whose session blob is missing. Non-zero usually means TTL mismatch.',+ labelNames: ['reason'], // 'event-time' | 'deadman'+});+register.registerMetric(sessionsReaperOrphans);++export const sessionDurationOnClose = new client.Histogram({+ name: 'session_duration_ms_on_close',+ help: 'Duration of closed sessions (ms)',+ buckets: [+ 1000, // 1s+ 10_000, // 10s+ 60_000, // 1m+ 5 * 60_000, // 5m+ 15 * 60_000, // 15m+ 30 * 60_000, // 30m+ 60 * 60_000, // 1h+ 24 * 60 * 60_000, // 24h+ ],+});+register.registerMetric(sessionDurationOnClose);++export const sessionEventsOnClose = new client.Histogram({+ name: 'session_events_on_close',+ help: 'Total events (event_count + screen_view_count) on session close',+ buckets: [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000],+});+register.registerMetric(sessionEventsOnClose);++// Gauges polled at scrape time. The active-sessions gauge does one ZCARD+// per project — cheap individually, fine for hundreds of projects.++register.registerMetric(+ new client.Gauge({+ name: 'sessions_active_total',+ help: 'Active sessions in Redis across all projects',+ async collect() {+ const redis = getRedisCache();+ const projectIds = await redis.smembers('session:projects');+ if (projectIds.length === 0) {+ this.set(0);+ return;+ }+ const multi = redis.multi();+ for (const pid of projectIds) {+ multi.zcard(`session:wallclock:${pid}`);+ }+ const results = await multi.exec();+ let total = 0;+ for (const entry of results ?? []) {+ const count = Number(entry?.[1] ?? 0);+ if (Number.isFinite(count)) {+ total += count;+ }+ }+ this.set(total);+ },+ })+);++register.registerMetric(+ new client.Gauge({+ name: 'sessions_projects_active',+ help: 'Projects with at least one active session',+ async collect() {+ const redis = getRedisCache();+ this.set(await redis.scard('session:projects'));+ },+ })+);++register.registerMetric(+ new client.Gauge({+ name: 'sessions_hwm_lag_ms',+ help: 'Max lag (ms) between wall-clock now and project event-time HWM, across all projects. Big number → queue lag or imports.',+ async collect() {+ const redis = getRedisCache();+ const projectIds = await redis.smembers('session:projects');+ if (projectIds.length === 0) {+ this.set(0);+ return;+ }+ const multi = redis.multi();+ for (const pid of projectIds) {+ multi.get(`session:hwm:${pid}`);+ }+ const results = await multi.exec();+ const now = Date.now();+ let maxLag = 0;+ for (const entry of results ?? []) {+ const hwm = Number(entry?.[1] ?? 0);+ if (!Number.isFinite(hwm) || hwm <= 0) {+ continue;+ }+ const lag = now - hwm;+ if (lag > maxLag) {+ maxLag = lag;+ }+ }+ this.set(maxLag);+ },+ })+);++export const sessionsVacuumed = new client.Counter({+ name: 'sessions_vacuumed_total',+ help: 'Sessions removed by the daily vacuum cron (catches blobs that cleanup() missed)',+ labelNames: ['reason'], // 'stale_blob' | 'missing_blob'+});+register.registerMetric(sessionsVacuumed);diff --git a/apps/worker/src/utils/session-handler.ts b/apps/worker/src/utils/session-handler.tsindex 0ca4c11ed..ecf43dd9f 100644--- a/apps/worker/src/utils/session-handler.ts+++ b/apps/worker/src/utils/session-handler.ts@@ -1,98 +1,58 @@-import type { IServiceCreateEventPayload } from '@openpanel/db';-import {- type EventsQueuePayloadCreateSessionEnd,- sessionsQueue,-} from '@openpanel/queue';-import { LRUCache } from '@openpanel/redis';-import type { Job } from 'bullmq';-import { logger as baseLogger } from '@/utils/logger';--export const SESSION_TIMEOUT = 1000 * 60 * 30;--const CHANGE_DELAY_THROTTLE_MS = process.env.CHANGE_DELAY_THROTTLE_MS- ? Number.parseInt(process.env.CHANGE_DELAY_THROTTLE_MS, 10)- : 60_000; // 1 minute--const CHANGE_DELAY_THROTTLE_MAP = new LRUCache<string, number>({- max: 100_000,- ttl: SESSION_TIMEOUT,-});--export type SessionEndJob = Job<EventsQueuePayloadCreateSessionEnd>;--// Treats any existing session-end job as "session in progress" regardless of-// state — avoids duplicate session_starts when the close job is mid-flight.-export async function getActiveSessionEndJob(- projectId: string,- deviceId: string,-): Promise<SessionEndJob | null> {- const jobId = getSessionEndJobId(projectId, deviceId);- const job = await sessionsQueue.getJob(jobId);- return (job as SessionEndJob | undefined) ?? null;-}--export async function extendSessionEndJob({- projectId,- deviceId,- job,-}: {- projectId: string;- deviceId: string;- job?: SessionEndJob | null;-}) {- const last = CHANGE_DELAY_THROTTLE_MAP.get(`${projectId}:${deviceId}`) ?? 0;- const isThrottled = Date.now() - last < CHANGE_DELAY_THROTTLE_MS;-- if (isThrottled) {- return;- }-- const resolvedJob =- job ?? (await getActiveSessionEndJob(projectId, deviceId));-- if (!resolvedJob) {- return;- }-- const state = await resolvedJob.getState();- if (state !== 'delayed') {- baseLogger.warn(- { jobId: resolvedJob.id, state },- 'Session end job is not in delayed state, skipping extend',- );- return;- }-- // Narrow TOCTOU window: state could flip from delayed → waiting between- // the getState() check above and this call.- try {- await resolvedJob.changeDelay(SESSION_TIMEOUT);- CHANGE_DELAY_THROTTLE_MAP.set(`${projectId}:${deviceId}`, Date.now());- } catch (error) {- baseLogger.warn(- { err: error, jobId: resolvedJob.id },- 'Session end job moved out of delayed state during extend',- );- }-}--const getSessionEndJobId = (projectId: string, deviceId: string) =>- `sessionEnd:${projectId}:${deviceId}`;--export function createSessionEndJob({+import type {+ IClickhouseSession,+ IServiceCreateEventPayload,+} from '@openpanel/db';+import { SESSION_TIMEOUT_MS } from '@openpanel/db';+import { sessionsQueue } from '@openpanel/queue';++export { SESSION_TIMEOUT_MS };+export const SESSION_TIMEOUT = SESSION_TIMEOUT_MS;++/**+ * Deterministic v2 jobId for a closed session.+ *+ * Keyed on the session's stable `id` so concurrent / retried closes for the+ * same logical session dedupe in BullMQ. The `v2:` prefix lets the legacy+ * drain script differentiate from pre-migration delayed jobs.+ *+ * Format constraint: BullMQ only accepts ':' in custom jobIds when splitting+ * by ':' yields exactly 3 parts, so we keep the suffix as a single segment.+ * Cross-project sessionInternalId collisions would be astronomical given the+ * 128-bit hash used to generate them.+ */+export const getSessionEndJobIdV2 = (sessionInternalId: string) =>+ `sessionEnd:v2:${sessionInternalId}`;++/**+ * Enqueue a session_end job. Idempotent via jobId.+ *+ * Called when a boundary is detected during ingest (old session must close)+ * or by the reaper when a session has been idle past the timeout.+ */+export async function enqueueSessionEndV2({payload,+ closedSession,}: {payload: IServiceCreateEventPayload;+ closedSession: IClickhouseSession;}) {+ const jobId = getSessionEndJobIdV2(closedSession.id);+return sessionsQueue.add('session',{type: 'createSessionEnd',- payload,+ payload: {+ ...payload,+ projectId: closedSession.project_id,+ deviceId: closedSession.device_id,+ sessionId: closedSession.id,+ profileId: closedSession.profile_id || payload.profileId,+ },+ snapshot: closedSession,},{- delay: SESSION_TIMEOUT,- jobId: getSessionEndJobId(payload.projectId, payload.deviceId),+ jobId,attempts: 3,backoff: {type: 'exponential',diff --git a/docker-compose.yml b/docker-compose.ymlindex c3468227f..189574789 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -61,3 +61,15 @@ services:ports:- "19092:19092" # Kafka API (external)- "9644:9644" # Admin API++ op-rp-console:+ image: redpandadata/console:v3.7.2+ restart: always+ depends_on:+ - op-rp+ environment:+ KAFKA_BROKERS: op-rp:9092+ REDPANDA_ADMINAPI_ENABLED: "true"+ REDPANDA_ADMINAPI_URLS: http://op-rp:9644+ ports:+ - "8091:8080"diff --git a/packages/db/scripts/check-sessions.ts b/packages/db/scripts/check-sessions.tsnew file mode 100644index 000000000..1a1cc8d7a--- /dev/null+++ b/packages/db/scripts/check-sessions.ts@@ -0,0 +1,424 @@+// Read-only health check for the session lifecycle pipeline.+//+// Inspects Redis state, the sessions BullMQ queue, both internal buffers,+// and ClickHouse — then reconciles them to surface anything stuck or+// drifting. Safe to run anytime; never mutates state.+//+// Usage:+// pnpm with-env tsx scripts/check-sessions.ts+// pnpm with-env tsx scripts/check-sessions.ts --verbose+// pnpm with-env tsx scripts/check-sessions.ts --project beatchurn-website+// pnpm with-env tsx scripts/check-sessions.ts --hours 48++import { getSafeJson } from '@openpanel/json';+import { sessionsQueue } from '@openpanel/queue';+import { getRedisCache } from '@openpanel/redis';+import { ch } from '../src/clickhouse/client';+import type { IClickhouseSession } from '../src/services/session.service';++const DEADMAN_MS = Number.parseInt(+ process.env.SESSION_REAPER_WALLCLOCK_DEADMAN_MS || String(30 * 60 * 1000),+ 10+);+const REAPER_INTERVAL_MS = 5 * 60 * 1000;+// "Eligible" = past the deadman, reaper is expected to close shortly.+// "Stuck" = past 3x reaper interval beyond the deadman — reaper is behind.+const STUCK_GRACE_MS = REAPER_INTERVAL_MS * 3;+const STUCK_CUTOFF_MS = DEADMAN_MS + STUCK_GRACE_MS;+const SAMPLE_SIZE_PER_PROJECT = 100;++type ProjectStat = {+ project: string;+ active: number;+ eligible: number;+ stuck: number;+ orphans: number;+ oldestAgeMin: number;+};++type ChCounts = {+ starts1h: number;+ ends1h: number;+ starts24h: number;+ ends24h: number;+};++function fmt(n: number, w: number) {+ return String(n).padStart(w);+}++function header(title: string) {+ console.log('');+ console.log('─'.repeat(70));+ console.log(title);+ console.log('─'.repeat(70));+}++async function inspectRedis(projects: string[]): Promise<{+ total: number;+ totalEligible: number;+ totalStuck: number;+ totalOrphans: number;+ stats: ProjectStat[];+}> {+ const redis = getRedisCache();+ const now = Date.now();+ const eligibleCutoff = now - DEADMAN_MS;+ const stuckCutoff = now - STUCK_CUTOFF_MS;++ const stats: ProjectStat[] = [];+ let total = 0;+ let totalEligible = 0;+ let totalStuck = 0;+ let totalOrphans = 0;++ for (const project of projects) {+ const wallclockKey = `session:wallclock:${project}`;++ const [active, eligible, stuck, oldestEntry] = await Promise.all([+ redis.zcard(wallclockKey),+ redis.zcount(wallclockKey, '-inf', eligibleCutoff),+ redis.zcount(wallclockKey, '-inf', stuckCutoff),+ redis.zrange(wallclockKey, 0, 0, 'WITHSCORES'),+ ]);++ const oldestAgeMin =+ oldestEntry.length === 2 ? (now - Number(oldestEntry[1])) / 60_000 : 0;++ // Sample N oldest entries and verify their blob exists.+ const sampleSize = Math.min(SAMPLE_SIZE_PER_PROJECT, active);+ let orphans = 0;+ if (sampleSize > 0) {+ const sample = await redis.zrange(wallclockKey, 0, sampleSize - 1);+ const multi = redis.multi();+ for (const did of sample) {+ multi.exists(`session:${project}:${did}`);+ }+ const results = await multi.exec();+ for (const entry of results ?? []) {+ if (Number(entry?.[1] ?? 0) === 0) orphans++;+ }+ }++ stats.push({ project, active, eligible, stuck, orphans, oldestAgeMin });+ total += active;+ totalEligible += eligible;+ totalStuck += stuck;+ totalOrphans += orphans;+ }++ return { total, totalEligible, totalStuck, totalOrphans, stats };+}++async function inspectQueue() {+ const [waiting, delayed, active, failed, completed] = await Promise.all([+ sessionsQueue.getWaitingCount(),+ sessionsQueue.getDelayedCount(),+ sessionsQueue.getActiveCount(),+ sessionsQueue.getFailedCount(),+ sessionsQueue.getCompletedCount(),+ ]);++ const failedSample = failed > 0 ? await sessionsQueue.getFailed(0, 4) : [];++ return { waiting, delayed, active, failed, completed, failedSample };+}++async function inspectBuffers() {+ const redis = getRedisCache();+ const [eventCounter, sessionCounter, eventListLen, sessionListLen] =+ await Promise.all([+ redis.get('buffer-counter:event'),+ redis.get('buffer-counter:session'),+ redis.llen('event-buffer'),+ redis.llen('session-buffer'),+ ]);++ return {+ event: {+ counter: Number(eventCounter ?? 0),+ listLen: Number(eventListLen ?? 0),+ },+ session: {+ counter: Number(sessionCounter ?? 0),+ listLen: Number(sessionListLen ?? 0),+ },+ };+}++async function inspectClickhouse(+ projects: string[] | null,+ hours: number+): Promise<Map<string, ChCounts>> {+ const projectClause = projects?.length+ ? `AND project_id IN (${projects.map((p) => `'${p.replace(/'/g, "''")}'`).join(',')})`+ : '';+ const query = `+ SELECT+ project_id,+ name,+ sum(if(created_at > now() - INTERVAL 1 HOUR, 1, 0)) AS h1,+ sum(if(created_at > now() - INTERVAL ${hours} HOUR, 1, 0)) AS hN+ FROM events+ WHERE name IN ('session_start', 'session_end')+ AND created_at > now() - INTERVAL ${hours} HOUR+ ${projectClause}+ GROUP BY project_id, name+ `;+ const res = await ch.query({ query, format: 'JSONEachRow' });+ const rows = await res.json<{+ project_id: string;+ name: string;+ h1: string;+ hN: string;+ }>();++ const byProject = new Map<string, ChCounts>();+ for (const r of rows) {+ const cur = byProject.get(r.project_id) ?? {+ starts1h: 0,+ ends1h: 0,+ starts24h: 0,+ ends24h: 0,+ };+ if (r.name === 'session_start') {+ cur.starts1h = Number(r.h1);+ cur.starts24h = Number(r.hN);+ } else {+ cur.ends1h = Number(r.h1);+ cur.ends24h = Number(r.hN);+ }+ byProject.set(r.project_id, cur);+ }+ return byProject;+}++async function inspectSessionEndClaims() {+ const redis = getRedisCache();+ let cursor = '0';+ let count = 0;+ do {+ const [next, batch] = await redis.scan(+ cursor,+ 'MATCH',+ 'session:end:emitted:*',+ 'COUNT',+ 1000+ );+ cursor = next;+ count += batch.length;+ } while (cursor !== '0' && count < 100_000);+ return count;+}++async function sampleOldestSession(+ projectId: string,+ deviceId: string+): Promise<IClickhouseSession | null> {+ const blob = await getRedisCache().get(`session:${projectId}:${deviceId}`);+ return blob ? getSafeJson<IClickhouseSession>(blob) : null;+}++async function main() {+ const args = process.argv.slice(2);+ const verbose = args.includes('--verbose');+ const projectIdx = args.indexOf('--project');+ const projectFilter = projectIdx >= 0 ? args[projectIdx + 1] : null;+ const hoursIdx = args.indexOf('--hours');+ const hours = hoursIdx >= 0 ? Number(args[hoursIdx + 1]) : 24;++ const redis = getRedisCache();++ console.log('═'.repeat(70));+ console.log('SESSION LIFECYCLE HEALTH CHECK');+ console.log(`time: ${new Date().toISOString()}`);+ console.log(`deadman: ${DEADMAN_MS / 60_000} min`);+ console.log(`stuck cutoff: ${STUCK_CUTOFF_MS / 60_000} min (deadman + 3 reaper ticks)`);+ console.log(`ch window: ${hours}h`);+ if (projectFilter) console.log(`project: ${projectFilter}`);+ console.log('═'.repeat(70));++ // 1. Redis inventory+ const allProjects = await redis.smembers('session:projects');+ const projects = projectFilter+ ? [projectFilter]+ : allProjects.slice().sort();++ header('REDIS — active sessions');+ console.log(` ${allProjects.length} projects in session:projects`);++ const redisInfo = await inspectRedis(projects);+ console.log(` ${redisInfo.total} total active sessions`);+ console.log(` ${redisInfo.totalEligible} past deadman (expected to reap soon)`);+ console.log(+ ` ${redisInfo.totalStuck} past deadman + ${STUCK_GRACE_MS / 60_000}min (stuck — reaper is behind)`+ );+ console.log(+ ` ${redisInfo.totalOrphans} wallclock entries with no blob (in sampled ${SAMPLE_SIZE_PER_PROJECT}/project)`+ );++ if (verbose && redisInfo.stats.length > 0) {+ console.log('');+ console.log(+ ` ${'project'.padEnd(40)} ${'active'.padStart(8)} ${'eligible'.padStart(10)} ${'stuck'.padStart(8)} ${'orphans'.padStart(8)} ${'oldest'.padStart(10)}`+ );+ for (const s of redisInfo.stats.sort((a, b) => b.active - a.active)) {+ console.log(+ ` ${s.project.padEnd(40)} ${fmt(s.active, 8)} ${fmt(s.eligible, 10)} ${fmt(s.stuck, 8)} ${fmt(s.orphans, 8)} ${s.oldestAgeMin.toFixed(1).padStart(8)}m`+ );+ }+ }++ // 2. Sessions queue+ header('SESSIONS QUEUE — createSessionEnd jobs');+ const queue = await inspectQueue();+ console.log(` waiting: ${queue.waiting}`);+ console.log(` delayed: ${queue.delayed}`);+ console.log(` active: ${queue.active}`);+ console.log(` failed: ${queue.failed}`);+ console.log(` completed: ${queue.completed} (removed-on-complete)`);+ if (queue.failedSample.length > 0) {+ console.log('');+ console.log(' Latest failed jobs:');+ for (const job of queue.failedSample) {+ const when = new Date(job.timestamp).toISOString();+ const reason = (job.failedReason ?? '').slice(0, 200);+ console.log(` - [${when}] ${job.id}`);+ console.log(` ${reason}`);+ }+ }++ // 3. Buffers+ header('BUFFERS — pending writes to ClickHouse');+ const buffers = await inspectBuffers();+ console.log(+ ` event buffer: counter=${buffers.event.counter} list=${buffers.event.listLen}`+ );+ console.log(+ ` session buffer: counter=${buffers.session.counter} list=${buffers.session.listLen}`+ );++ // 4. Dedup claims+ header('IDEMPOTENCY — session:end:emitted:* claim keys');+ const claimCount = await inspectSessionEndClaims();+ console.log(+ ` ${claimCount} active claim keys (TTL 2h; bounds at ~last 2h of closes)`+ );++ // 5. ClickHouse reconciliation+ header(`CLICKHOUSE — session_start vs session_end (last ${hours}h)`);+ const chCounts = await inspectClickhouse(+ projectFilter ? [projectFilter] : null,+ hours+ );++ let totalStarts1h = 0;+ let totalEnds1h = 0;+ let totalStartsN = 0;+ let totalEndsN = 0;++ console.log('');+ console.log(+ ` ${'project'.padEnd(40)} ${'start 1h'.padStart(9)} ${'end 1h'.padStart(8)} ${'start ' + hours + 'h'}`.padEnd(58) ++ `${('end ' + hours + 'h').padStart(8)} ${'open'.padStart(8)}`+ );+ for (const [project, c] of [...chCounts.entries()].sort()) {+ const open = c.starts24h - c.ends24h;+ console.log(+ ` ${project.padEnd(40)} ${fmt(c.starts1h, 9)} ${fmt(c.ends1h, 8)} ${fmt(c.starts24h, 10)} ${fmt(c.ends24h, 8)} ${fmt(open, 8)}`+ );+ totalStarts1h += c.starts1h;+ totalEnds1h += c.ends1h;+ totalStartsN += c.starts24h;+ totalEndsN += c.ends24h;+ }+ const openInCh = totalStartsN - totalEndsN;+ console.log('');+ console.log(+ ` ${'TOTAL'.padEnd(40)} ${fmt(totalStarts1h, 9)} ${fmt(totalEnds1h, 8)} ${fmt(totalStartsN, 10)} ${fmt(totalEndsN, 8)} ${fmt(openInCh, 8)}`+ );++ // 6. Reconciliation+ header('RECONCILIATION — Redis active vs CH open');+ const diff = redisInfo.total - openInCh;+ const diffPct =+ openInCh > 0 ? ((Math.abs(diff) / openInCh) * 100).toFixed(1) : '∞';+ console.log(` Redis active sessions: ${redisInfo.total}`);+ console.log(` CH starts - ends in ${hours}h window: ${openInCh}`);+ console.log(` Diff: ${diff} (${diffPct}%)`);+ console.log(+ ` Note: diff > 0 means some "open" sessions in Redis were started before the ${hours}h window.`+ );++ // 7. Sample the oldest session in Redis if anything looks suspicious+ if (verbose && redisInfo.stats.some((s) => s.oldestAgeMin > 60 * 6)) {+ header('SAMPLE — oldest session per project (any > 6h)');+ for (const s of redisInfo.stats.filter((x) => x.oldestAgeMin > 60 * 6)) {+ const wallclockKey = `session:wallclock:${s.project}`;+ const [oldest] = await redis.zrange(wallclockKey, 0, 0);+ if (oldest) {+ const blob = await sampleOldestSession(s.project, oldest);+ if (blob) {+ console.log(` ${s.project} / ${oldest}`);+ console.log(` id: ${blob.id}`);+ console.log(` created_at: ${blob.created_at}`);+ console.log(` ended_at: ${blob.ended_at}`);+ console.log(` event_count: ${blob.event_count}`);+ }+ }+ }+ }++ // 8. Summary+ header('SUMMARY');+ const issues: string[] = [];+ if (redisInfo.totalStuck > 0) {+ issues.push(+ `${redisInfo.totalStuck} sessions past the stuck cutoff — reaper may be unhealthy`+ );+ }+ if (redisInfo.totalOrphans > 0) {+ issues.push(+ `${redisInfo.totalOrphans} orphan wallclock entries — cleanup() raced or vacuum is overdue`+ );+ }+ if (queue.failed > 0) {+ issues.push(`${queue.failed} failed session_end jobs`);+ }+ if (queue.waiting + queue.active > 5000) {+ issues.push(+ `session_end backlog (waiting=${queue.waiting} active=${queue.active})`+ );+ }+ if (buffers.event.listLen > 50_000) {+ issues.push(`event buffer is backed up (${buffers.event.listLen} items)`);+ }+ if (buffers.session.listLen > 50_000) {+ issues.push(+ `session buffer is backed up (${buffers.session.listLen} items)`+ );+ }+ // The Redis-vs-CH diff is INFORMATIONAL, not an issue on its own:+ // diff > 0: more in Redis than CH's window — older sessions still active+ // diff < 0: more starts in CH than Redis active — legacy starts without ends+ // (e.g. from before a bug fix) that will never close+ // Only flag if both reaper is behind AND the diff is growing — which we can't+ // know from a single run. So just print it; the reaper-behind and stuck+ // counters above will catch live issues.++ if (issues.length === 0) {+ console.log(' [OK] No issues detected.');+ } else {+ console.log(' [WARN] Issues:');+ for (const i of issues) console.log(` - ${i}`);+ }++ await redis.quit();+ await ch.close?.();+ process.exit(issues.length > 0 ? 1 : 0);+}++main().catch((e) => {+ console.error(e);+ process.exit(1);+});diff --git a/packages/db/scripts/drain-old-session-jobs.ts b/packages/db/scripts/drain-old-session-jobs.tsnew file mode 100644index 000000000..3fd539170--- /dev/null+++ b/packages/db/scripts/drain-old-session-jobs.ts@@ -0,0 +1,79 @@+// One-off cleanup: drain legacy `sessionEnd:{projectId}:{deviceId}` delayed+// jobs from the `sessions` queue. Run AFTER:+//+// 1. The new code is deployed (worker emits reaper-enqueued jobs with the+// `sessionEnd:v2:*` jobId prefix).+// 2. The `events.incoming-event` queue has been resumed and you've verified+// session metrics look healthy under the new flow.+//+// This removes only jobs whose jobId is `sessionEnd:*` and NOT `sessionEnd:v2:*`,+// leaving any reaper-enqueued v2 jobs intact. After this script runs you can+// safely resume the `sessions` queue.+//+// Idempotent: re-running on an already-drained queue is a no-op.+//+// Usage (from packages/db):+// pnpm with-env tsx scripts/drain-old-session-jobs.ts+// pnpm with-env tsx scripts/drain-old-session-jobs.ts --dry-run++import { sessionsQueue } from '@openpanel/queue';++async function main() {+ const dryRun = process.argv.includes('--dry-run');++ console.log(+ `[drain-old-session-jobs] starting${dryRun ? ' (DRY RUN — no removes)' : ''}`+ );++ const jobs = await sessionsQueue.getJobs(['delayed', 'waiting']);++ const legacy: typeof jobs = [];+ const v2: typeof jobs = [];++ for (const job of jobs) {+ if (!job.id) continue;+ if (job.id.startsWith('sessionEnd:v2:')) {+ v2.push(job);+ } else if (job.id.startsWith('sessionEnd:')) {+ legacy.push(job);+ }+ }++ console.log(+ `[drain-old-session-jobs] found ${jobs.length} jobs total ` ++ `(${legacy.length} legacy, ${v2.length} v2, will keep v2)`+ );++ if (dryRun) {+ console.log('[drain-old-session-jobs] dry run — exiting');+ process.exit(0);+ }++ let removed = 0;+ let failed = 0;++ for (const job of legacy) {+ try {+ await job.remove();+ removed++;+ if (removed % 1000 === 0) {+ console.log(`[drain-old-session-jobs] removed ${removed}/${legacy.length}`);+ }+ } catch (error) {+ failed++;+ console.error('[drain-old-session-jobs] failed to remove job', job.id, error);+ }+ }++ console.log('[drain-old-session-jobs] done', {+ removed,+ failed,+ kept_v2: v2.length,+ });+ process.exit(0);+}++main().catch((error) => {+ console.error('[drain-old-session-jobs] fatal error:', error);+ process.exit(1);+});diff --git a/packages/db/scripts/migrate-sessions.ts b/packages/db/scripts/migrate-sessions.tsnew file mode 100644index 000000000..ca991a023--- /dev/null+++ b/packages/db/scripts/migrate-sessions.ts@@ -0,0 +1,121 @@+// One-off migration: convert legacy `session:{sessionId}` Redis blobs ++// delayed BullMQ `sessionEnd:{projectId}:{deviceId}` jobs into the new+// session-buffer key scheme:+//+// session:{projectId}:{deviceId} -> full session JSON blob (no TTL)+// session:profile:{projectId}:{profileId} -> deviceId pointer (no TTL)+// session:wallclock:{projectId} -> ZSET scored by wall-clock (ms)+// session:projects -> SET of project_ids with active sessions+//+// Source of truth: the delayed `sessionsQueue` jobs — one per (projectId,+// deviceId) pair currently considered "active" by the legacy system. For+// each, we read the corresponding `session:{sessionId}` blob (still alive+// under its 1h TTL) and rewrite it into the new structure.+//+// Idempotent: re-running the script is safe. Existing new-scheme keys are+// preserved via SET NX semantics inside the multi pipeline.+//+// Usage (from packages/db, env loaded from repo root .env):+// pnpm with-env tsx scripts/migrate-sessions.ts+// pnpm with-env tsx scripts/migrate-sessions.ts --dry-run+//+// Runbook step 2: pause `events.incoming-event` AND `sessions` queues first,+// wait for in-flight jobs to drain (~30s), THEN run this script. Resume+// `events.incoming-event` after deploy; keep `sessions` paused until step 7+// (drain-old-session-jobs.ts) finishes.++import { getSafeJson } from '@openpanel/json';+import {+ type EventsQueuePayloadCreateSessionEnd,+ sessionsQueue,+} from '@openpanel/queue';+import { getRedisCache } from '@openpanel/redis';+import type { IClickhouseSession } from '../src/services/session.service';++const sessionKey = (projectId: string, deviceId: string) =>+ `session:${projectId}:${deviceId}`;+const profileIndexKey = (projectId: string, profileId: string) =>+ `session:profile:${projectId}:${profileId}`;+const wallclockSetKey = (projectId: string) =>+ `session:wallclock:${projectId}`;+const PROJECTS_SET_KEY = 'session:projects';++async function main() {+ const dryRun = process.argv.includes('--dry-run');+ const redis = getRedisCache();++ console.log(+ `[migrate-sessions] starting${dryRun ? ' (DRY RUN — no writes)' : ''}`+ );++ const jobs = await sessionsQueue.getJobs(['delayed', 'waiting']);+ console.log(`[migrate-sessions] found ${jobs.length} legacy session jobs`);++ const counts = {+ migrated: 0,+ blobMissing: 0,+ alreadyMigrated: 0,+ parseFailed: 0,+ };++ for (const job of jobs) {+ const data = job.data as EventsQueuePayloadCreateSessionEnd | undefined;+ const payload = data?.payload;+ if (!payload?.projectId || !payload.deviceId || !payload.sessionId) {+ counts.parseFailed++;+ continue;+ }+ const { projectId, deviceId, sessionId } = payload;++ const existsAlready = await redis.exists(sessionKey(projectId, deviceId));+ if (existsAlready) {+ counts.alreadyMigrated++;+ continue;+ }++ const blob = await redis.get(`session:${sessionId}`);+ if (!blob) {+ counts.blobMissing++;+ continue;+ }++ const session = getSafeJson<IClickhouseSession>(blob);+ if (!session) {+ counts.parseFailed++;+ continue;+ }++ const nowMs = Date.now();++ if (dryRun) {+ counts.migrated++;+ continue;+ }++ const multi = redis.multi();+ multi.set(sessionKey(projectId, deviceId), blob);+ multi.zadd(wallclockSetKey(projectId), nowMs.toString(), deviceId);+ multi.sadd(PROJECTS_SET_KEY, projectId);+ if (session.profile_id && session.profile_id !== deviceId) {+ multi.set(profileIndexKey(projectId, session.profile_id), deviceId);+ }+ await multi.exec();++ counts.migrated++;++ if (counts.migrated % 1000 === 0) {+ console.log(+ `[migrate-sessions] progress: ${counts.migrated} migrated`+ );+ }+ }++ console.log('[migrate-sessions] done', counts);+ await redis.quit();+ process.exit(0);+}++main().catch((error) => {+ console.error('[migrate-sessions] fatal error:', error);+ process.exit(1);+});diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex ff8634890..5c279788a 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -153,7 +153,7 @@ export class BaseBuffer {*/protected getYieldInterval(batchSize: number,- opts: { min?: number; max?: number; targetYieldsPerFlush?: number } = {},+ opts: { min?: number; max?: number; targetYieldsPerFlush?: number } = {}): number {const target = opts.targetYieldsPerFlush ?? 20;const min = opts.min ?? 100;@@ -183,7 +183,7 @@ export class BaseBuffer {for (const line of lines) {yield line;}- })(),+ })());}@@ -371,7 +371,7 @@ export class BaseBuffer {} else if (obs.result === 'locked') {this.logger.debug(logPayload, `Flush skipped for ${this.name} (locked)`);} else {- this.logger.info(logPayload, `Flush completed for ${this.name}`);+ this.logger.debug(logPayload, `Flush completed for ${this.name}`);}}diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.tsindex 86741b540..46b5920bc 100644--- a/packages/db/src/buffers/index.ts+++ b/packages/db/src/buffers/index.ts@@ -16,3 +16,7 @@ export const groupBuffer = new GroupBuffer();export type { ProfileBackfillEntry } from './profile-backfill-buffer';export type { IClickhouseSessionReplayChunk } from './replay-buffer';+export {+ SESSION_TIMEOUT_MS,+ type SessionIngestResult,+} from './session-buffer';diff --git a/packages/db/src/buffers/session-buffer.test.ts b/packages/db/src/buffers/session-buffer.test.tsindex f27f0170d..e6ba88bca 100644--- a/packages/db/src/buffers/session-buffer.test.ts+++ b/packages/db/src/buffers/session-buffer.test.ts@@ -11,38 +11,61 @@ vi.mock('../clickhouse/client', () => ({},}));+import type { IServiceCreateEventPayload } from '../services/event.service';import { SessionBuffer } from './session-buffer';-import type { IClickhouseEvent } from '../services/event.service';const redis = getRedisCache();-function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {+const projectId = 'project-1';+const deviceId = 'device-1';+const sessionId = 'session-1';++function makePayload(+ overrides: Partial<IServiceCreateEventPayload> = {}+): IServiceCreateEventPayload {return {- id: 'event-1',- project_id: 'project-1',- profile_id: 'profile-1',- device_id: 'device-1',- session_id: 'session-1',name: 'screen_view',+ projectId,+ deviceId,+ sessionId,+ profileId: 'profile-1',+ properties: {},+ groups: [],+ createdAt: new Date(),+ duration: 0,+ sdkName: 'web',+ sdkVersion: '1.0.0',+ city: '',+ country: '',+ region: '',+ longitude: 0,+ latitude: 0,path: '/home',origin: '',referrer: '',- referrer_name: '',- referrer_type: '',- duration: 0,- properties: {},- created_at: new Date().toISOString(),- groups: [],+ referrerName: '',+ referrerType: '',+ os: '',+ osVersion: '',+ browser: '',+ browserVersion: '',+ device: '',+ brand: '',+ model: '',...overrides,- } as IClickhouseEvent;+ };}beforeEach(async () => {const keys = [- ...await redis.keys('session*'),- ...await redis.keys('lock:session'),+ ...(await redis.keys('session*')),+ ...(await redis.keys('lock:session')),];- if (keys.length > 0) await redis.del(...keys);+ // Spread breaks at ~100k args — del in chunks so old test state can't kill us.+ const BATCH = 1000;+ for (let i = 0; i < keys.length; i += BATCH) {+ await redis.del(...keys.slice(i, i + BATCH));+ }vi.mocked(ch.insert).mockResolvedValue(undefined as any);});@@ -59,37 +82,217 @@ describe('SessionBuffer', () => {sessionBuffer = new SessionBuffer();});- it('adds a new session to the buffer', async () => {- const sizeBefore = await sessionBuffer.getBufferSize();- await sessionBuffer.add(makeEvent({}));- const sizeAfter = await sessionBuffer.getBufferSize();+ it('opens a new session and stores it at session:{pid}:{did}', async () => {+ const result = await sessionBuffer.ingest(makePayload());++ expect(result?.kind).toBe('new');++ const blob = await redis.get(`session:${projectId}:${deviceId}`);+ expect(blob).not.toBeNull();+ const parsed = JSON.parse(blob ?? '');+ expect(parsed.id).toBe(sessionId);+ expect(parsed.project_id).toBe(projectId);+ expect(parsed.device_id).toBe(deviceId);+ });++ it('registers the device in the wallclock sorted set and the projects set', async () => {+ await sessionBuffer.ingest(makePayload());++ expect(+ await redis.zscore(`session:wallclock:${projectId}`, deviceId)+ ).not.toBeNull();+ expect(await redis.sismember('session:projects', projectId)).toBe(1);+ });++ it('writes no TTL on the session blob or profile pointer', async () => {+ await sessionBuffer.ingest(+ makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })+ );++ // Blob and profile pointer must survive arbitrarily long reaper outages,+ // so neither carries a Redis TTL.+ expect(await redis.ttl(`session:${projectId}:device-Y`)).toBe(-1);+ expect(await redis.ttl(`session:profile:${projectId}:profile-X`)).toBe(-1);+ });- expect(sizeAfter).toBe(sizeBefore + 1);+ it('writes profile→device pointer when profile_id differs from device_id', async () => {+ await sessionBuffer.ingest(+ makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })+ );+ expect(await redis.get(`session:profile:${projectId}:profile-X`)).toBe(+ 'device-Y'+ );});it('skips session_start and session_end events', async () => {const sizeBefore = await sessionBuffer.getBufferSize();- await sessionBuffer.add(makeEvent({ name: 'session_start' }));- await sessionBuffer.add(makeEvent({ name: 'session_end' }));- const sizeAfter = await sessionBuffer.getBufferSize();+ expect(+ await sessionBuffer.ingest(makePayload({ name: 'session_start' }))+ ).toBeNull();+ expect(+ await sessionBuffer.ingest(makePayload({ name: 'session_end' }))+ ).toBeNull();+ expect(await sessionBuffer.getBufferSize()).toBe(sizeBefore);+ });++ it('emits sign=-1 + sign=+1 CH rows when extending within timeout', async () => {+ const t0 = new Date();+ await sessionBuffer.ingest(makePayload({ createdAt: t0 }));+ const sizeBefore = await sessionBuffer.getBufferSize();- expect(sizeAfter).toBe(sizeBefore);+ const result = await sessionBuffer.ingest(+ makePayload({ createdAt: new Date(t0.getTime() + 5000) })+ );++ expect(result?.kind).toBe('extend');+ expect(await sessionBuffer.getBufferSize()).toBe(sizeBefore + 2);});- it('updates existing session on subsequent events', async () => {- const t0 = Date.now();- await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));+ it('returns kind=boundary when gap > 30min, with closed + current populated', async () => {+ const t0 = new Date();+ const first = await sessionBuffer.ingest(makePayload({ createdAt: t0 }));+ expect(first?.kind).toBe('new');- // Second event updates the same session — emits old (sign=-1) + new (sign=1)- const sizeBefore = await sessionBuffer.getBufferSize();- await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));- const sizeAfter = await sessionBuffer.getBufferSize();+ const second = await sessionBuffer.ingest(+ makePayload({+ createdAt: new Date(t0.getTime() + 35 * 60 * 1000),+ sessionId: 'session-2',+ })+ );- expect(sizeAfter).toBe(sizeBefore + 2);+ expect(second?.kind).toBe('boundary');+ if (second?.kind === 'boundary') {+ expect(second.closed.id).toBe(sessionId);+ expect(second.current.id).toBe('session-2');+ }+ });++ it('invariant: one visit keeps one id across extends; a >30min gap splits and closes the first', async () => {+ // This ties the layers together against real Redis: ingest() drives the+ // state machine, and getExistingSession() is what the API reads on each+ // request to resolve the id — they must agree throughout a visit.+ const t0 = new Date('2026-06-08T12:00:00.000Z');+ const at = (mins: number) => new Date(t0.getTime() + mins * 60_000);++ const e1 = await sessionBuffer.ingest(+ makePayload({ createdAt: t0, sessionId: 'S1' })+ );+ expect(e1?.kind).toBe('new');+ expect(e1?.current.id).toBe('S1');+ // The API's next lookup must see the live session.+ expect(+ (await sessionBuffer.getExistingSession({ projectId, deviceId }))?.id+ ).toBe('S1');++ // Events within the idle window extend the SAME session.+ for (const mins of [10, 20]) {+ const ext = await sessionBuffer.ingest(+ makePayload({ createdAt: at(mins), sessionId: 'S1' })+ );+ expect(ext?.kind).toBe('extend');+ expect(ext?.current.id).toBe('S1');+ }++ // A >30min gap: ingest detects the boundary, closes S1, opens S2.+ const e4 = await sessionBuffer.ingest(+ makePayload({ createdAt: at(55), sessionId: 'S2' })+ );+ expect(e4?.kind).toBe('boundary');+ if (e4?.kind === 'boundary') {+ expect(e4.closed.id).toBe('S1');+ expect(e4.current.id).toBe('S2');+ }++ // The slot now holds the new session — the API reads S2 from here on.+ expect(+ (await sessionBuffer.getExistingSession({ projectId, deviceId }))?.id+ ).toBe('S2');+ });++ it('inherits utm_* fields from event.properties.__query', async () => {+ await sessionBuffer.ingest(+ makePayload({+ properties: {+ __query: {+ utm_medium: 'cpc',+ utm_source: 'google',+ utm_campaign: 'spring',+ },+ },+ })+ );+ const blob = JSON.parse(+ (await redis.get(`session:${projectId}:${deviceId}`)) ?? ''+ );+ expect(blob.utm_medium).toBe('cpc');+ expect(blob.utm_source).toBe('google');+ expect(blob.utm_campaign).toBe('spring');+ });++ it('out-of-order events: older event extends backwards, never produces negative duration', async () => {+ const t0 = new Date();+ await sessionBuffer.ingest(makePayload({ createdAt: t0 }));+ // Late-arriving event with an older timestamp.+ await sessionBuffer.ingest(+ makePayload({ createdAt: new Date(t0.getTime() - 30_000) })+ );+ const blob = JSON.parse(+ (await redis.get(`session:${projectId}:${deviceId}`)) ?? ''+ );+ expect(blob.duration).toBeGreaterThanOrEqual(0);+ });++ it('cleanup() removes blob, wallclock entry, and profile pointer', async () => {+ await sessionBuffer.ingest(+ makePayload({ profileId: 'profile-X', deviceId: 'device-Y' })+ );++ await sessionBuffer.cleanup({+ projectId,+ deviceId: 'device-Y',+ sessionId,+ profileId: 'profile-X',+ });++ expect(await redis.get(`session:${projectId}:device-Y`)).toBeNull();+ expect(+ await redis.zscore(`session:wallclock:${projectId}`, 'device-Y')+ ).toBeNull();+ expect(+ await redis.get(`session:profile:${projectId}:profile-X`)+ ).toBeNull();+ });++ it('cleanup() is a no-op when slot is owned by a different session (boundary)', async () => {+ // Open session S1+ await sessionBuffer.ingest(makePayload({ sessionId: 's1' }));+ // 35min later — boundary, opens S2 at same (pid, did)+ await sessionBuffer.ingest(+ makePayload({+ sessionId: 's2',+ createdAt: new Date(Date.now() + 35 * 60 * 1000),+ })+ );++ // Try to cleanup S1 — should not touch S2's data. The Lua script in+ // cleanup() detects the id mismatch and returns early.+ await sessionBuffer.cleanup({+ projectId,+ deviceId,+ sessionId: 's1',+ });++ const blob = await redis.get(`session:${projectId}:${deviceId}`);+ expect(blob).not.toBeNull();+ const parsed = JSON.parse(blob ?? '');+ expect(parsed.id).toBe('s2');+ expect(+ await redis.zscore(`session:wallclock:${projectId}`, deviceId)+ ).not.toBeNull();});it('processes buffer and inserts sessions into ClickHouse', async () => {- await sessionBuffer.add(makeEvent({}));+ await sessionBuffer.ingest(makePayload());const insertSpy = vi.spyOn(ch, 'insert')@@ -107,15 +310,15 @@ describe('SessionBuffer', () => {it('squash does not orphan the creation row when create + updates share a batch', async () => {// Regression: a brand-new session whose creation event and subsequent- // updates all land in one flush window. getSession emits (+1,v1) for the+ // updates all land in one flush window. ingest emits (+1,v1) for the// creation then (-1,vN)/(+1,vN+1) pairs for each update. The squash must// net per version so nothing is left un-collapsible: a (-1,V) row only// collapses against a (+1,V) of the SAME version in CH.const t0 = Date.now();const EVENTS = 8;for (let i = 0; i < EVENTS; i++) {- await sessionBuffer.add(- makeEvent({ created_at: new Date(t0 + i * 1000).toISOString() }),+ await sessionBuffer.ingest(+ makePayload({ createdAt: new Date(t0 + i * 1000) }));}@@ -131,6 +334,13 @@ describe('SessionBuffer', () => {await sessionBuffer.processBuffer();+ // Squash must actually fire. The 8 events queue 1 creation + 7+ // (-1,vN)/(+1,vN+1) pairs = 15 raw rows; for a fresh session with no+ // CH-resident row they all net down to the single final (+1, v8). If+ // squash is disabled/removed, every raw row is inserted and this is 15+ // — this assertion is the guard that the optimisation is wired in.+ expect(inserted.length).toBe(1);+// Net the inserted rows per version exactly as CH's// VersionedCollapsingMergeTree(sign, version) would: a row survives only// if positives and negatives at its version don't cancel.@@ -148,7 +358,7 @@ describe('SessionBuffer', () => {});it('retains sessions in queue when ClickHouse insert fails', async () => {- await sessionBuffer.add(makeEvent({}));+ await sessionBuffer.ingest(makePayload());const insertSpy = vi.spyOn(ch, 'insert')@@ -158,7 +368,7 @@ describe('SessionBuffer', () => {// counter). processBuffer no longer swallows — we still verify the// safety property: the queue is preserved.await expect(sessionBuffer.processBuffer()).rejects.toThrow(- 'ClickHouse unavailable',+ 'ClickHouse unavailable');expect(await sessionBuffer.getBufferSize()).toBe(1);diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.tsindex 01f745c0c..8ff03ac63 100644--- a/packages/db/src/buffers/session-buffer.ts+++ b/packages/db/src/buffers/session-buffer.ts@@ -1,11 +1,74 @@+import { DateTime } from '@openpanel/common';import { getSafeJson } from '@openpanel/json';import { getRedisCache, type Redis } from '@openpanel/redis';-import { assocPath, clone } from 'ramda';import { ch, TABLE_NAMES } from '../clickhouse/client';-import type { IClickhouseEvent } from '../services/event.service';+import type { IServiceCreateEventPayload } from '../services/event.service';import type { IClickhouseSession } from '../services/session.service';import { BaseBuffer } from './base-buffer';+// 30min of idle in event-time → session ends. Matches industry default.+// Idle window for a session (boundary detection + the reaper's default deadman).+// Env-overridable so E2E tests can shrink it from 30min to a few seconds.+export const SESSION_TIMEOUT_MS = Number.parseInt(+ process.env.SESSION_TIMEOUT_MS || String(1000 * 60 * 30),+ 10+);++const sessionKey = (projectId: string, deviceId: string) =>+ `session:${projectId}:${deviceId}`;+const profileIndexKey = (projectId: string, profileId: string) =>+ `session:profile:${projectId}:${profileId}`;+const wallclockSetKey = (projectId: string) =>+ `session:wallclock:${projectId}`;+const PROJECTS_SET_KEY = 'session:projects';++// Atomic id-gated cleanup. Only deletes the blob + sorted-set entry ++// profile pointer if the live blob's session id matches the one we're+// closing. Prevents races where a boundary has already overwritten the slot+// with a new session.+const CLEANUP_LUA = `+local cur = redis.call('GET', KEYS[1])+if cur then+ local ok, parsed = pcall(cjson.decode, cur)+ if ok and parsed and parsed.id ~= ARGV[1] then+ return 0+ end+end+redis.call('DEL', KEYS[1])+redis.call('ZREM', KEYS[2], ARGV[2])+if KEYS[3] ~= '' then+ redis.call('DEL', KEYS[3])+end+return 1+`;++export type SessionIngestResult =+ | { kind: 'new'; current: IClickhouseSession }+ | { kind: 'extend'; current: IClickhouseSession }+ | { kind: 'boundary'; current: IClickhouseSession; closed: IClickhouseSession };++function toClickhouseDate(date: Date): string {+ return DateTime.fromJSDate(date)+ .setZone('UTC')+ .toFormat('yyyy-MM-dd HH:mm:ss.SSS');+}++function fromClickhouseDate(s: string): Date {+ return DateTime.fromFormat(s, 'yyyy-MM-dd HH:mm:ss.SSS', {+ zone: 'UTC',+ }).toJSDate();+}++function pickUtm(+ payload: IServiceCreateEventPayload,+ key: 'utm_medium' | 'utm_source' | 'utm_campaign' | 'utm_content' | 'utm_term'+): string {+ const query = (payload.properties as { __query?: Record<string, unknown> } | undefined)+ ?.__query;+ const v = query?.[key];+ return v ? String(v) : '';+}+export class SessionBuffer extends BaseBuffer {private batchSize = process.env.SESSION_BUFFER_BATCH_SIZE? Number.parseInt(process.env.SESSION_BUFFER_BATCH_SIZE, 10)@@ -30,226 +93,280 @@ export class SessionBuffer extends BaseBuffer {this.redis = getRedisCache();}+ /**+ * Look up the active session for a device, either directly by deviceId or+ * via the profile index (`session:profile:{pid}:{profileId}` → deviceId).+ */public async getExistingSession(options:- | {- sessionId: string;- }- | {- projectId: string;- profileId: string;- }- ) {- let hit: string | null = null;- if ('sessionId' in options) {- hit = await this.redis.get(`session:${options.sessionId}`);- } else {- const value = await this.redis.get(- `session:${options.projectId}:${options.profileId}`+ | { projectId: string; deviceId: string }+ | { projectId: string; profileId: string }+ ): Promise<IClickhouseSession | null> {+ if ('deviceId' in options) {+ const hit = await this.redis.get(+ sessionKey(options.projectId, options.deviceId));- if (!value) {- return null;- }-- // Backward compat: old keys stored full JSON, new keys store just the sessionId- if (value.startsWith('{')) {- return getSafeJson<IClickhouseSession>(value);- }-- hit = await this.redis.get(`session:${value}`);+ return hit ? getSafeJson<IClickhouseSession>(hit) : null;}+ const deviceId = await this.redis.get(+ profileIndexKey(options.projectId, options.profileId)+ );+ if (!deviceId) return null;+ const hit = await this.redis.get(sessionKey(options.projectId, deviceId));+ return hit ? getSafeJson<IClickhouseSession>(hit) : null;+ }- if (hit) {- return getSafeJson<IClickhouseSession>(hit);+ /**+ * Ingest one event into the session lifecycle.+ *+ * Reads the device's current session (if any), decides whether the event+ * extends it, opens a brand-new one, or boundaries (gap > 30min) → close+ * the old session and start a fresh one. Writes the updated session blob,+ * updates the wall-clock index, and queues CollapsingMergeTree rows for+ * ClickHouse.+ *+ * Returns the action taken so the caller can drive session_start /+ * session_end event emission. `session_start` / `session_end` events+ * themselves are skipped — they are derived signals, not session activity.+ */+ async ingest(+ payload: IServiceCreateEventPayload+ ): Promise<SessionIngestResult | null> {+ if (!payload.projectId || !payload.deviceId) return null;+ if (payload.name === 'session_start' || payload.name === 'session_end') {+ return null;}- return null;- }-- async getSession(- event: IClickhouseEvent- ): Promise<[IClickhouseSession] | [IClickhouseSession, IClickhouseSession]> {- const existingSession = await this.getExistingSession({- sessionId: event.session_id,- });+ try {+ const existing = await this.getExistingSession({+ projectId: payload.projectId,+ deviceId: payload.deviceId,+ });- if (existingSession) {- const oldSession = assocPath(['sign'], -1, clone(existingSession));- const newSession = assocPath(['sign'], 1, clone(existingSession));+ const eventTimeMs = payload.createdAt.getTime();+ const isBoundary =+ existing &&+ eventTimeMs - fromClickhouseDate(existing.ended_at).getTime() >+ SESSION_TIMEOUT_MS;- newSession.version = existingSession.version + 1;+ if (existing && !isBoundary) {+ const { current, chRows } = this.extendSession(existing, payload);+ await this.persist(current, chRows);+ return { kind: 'extend', current };+ }- // Events can arrive out of order (client-side batching, retries, offline- // queueing). Treat the session window as [min(event ts), max(event ts)]- // so duration stays non-negative and entry/exit reflect actual order.- const eventTime = new Date(event.created_at).getTime();- const startTime = new Date(newSession.created_at).getTime();- const endTime = new Date(newSession.ended_at).getTime();+ const current = this.newSession(payload);+ await this.persist(current, [current]);- if (eventTime >= endTime) {- newSession.ended_at = event.created_at;- if (event.path) {- newSession.exit_path = event.path;- }- if (event.origin) {- newSession.exit_origin = event.origin;- }+ if (existing && isBoundary) {+ return { kind: 'boundary', current, closed: existing };}+ return { kind: 'new', current };+ } catch (error) {+ this.logger.error({ err: error }, 'Failed to ingest session');+ return null;+ }+ }- if (eventTime < startTime) {- newSession.created_at = event.created_at;- if (event.path) {- newSession.entry_path = event.path;- }- if (event.origin) {- newSession.entry_origin = event.origin;- }- } else {- if (!newSession.entry_path && event.path) {- newSession.entry_path = event.path;- }- if (!newSession.entry_origin && event.origin) {- newSession.entry_origin = event.origin;- }- }+ /**+ * Remove a closed session from Redis. Atomically gated on `sessionId` via+ * Lua: if a different session now occupies the `(projectId, deviceId)`+ * slot (because a boundary already overwrote it with a new session), the+ * call is a no-op — the new session still needs its blob and sorted-set+ * entry.+ *+ * Blobs no longer have a Redis TTL — this is the only path that removes+ * them. The daily vacuum cron is a backstop for the rare case where this+ * fails (worker crash, network partition).+ */+ async cleanup({+ projectId,+ deviceId,+ sessionId,+ profileId,+ }: {+ projectId: string;+ deviceId: string;+ sessionId: string;+ profileId?: string | null;+ }): Promise<void> {+ const pointerKey =+ profileId && profileId !== deviceId+ ? profileIndexKey(projectId, profileId)+ : '';++ await this.redis.eval(+ CLEANUP_LUA,+ 3,+ sessionKey(projectId, deviceId),+ wallclockSetKey(projectId),+ pointerKey,+ sessionId,+ deviceId+ );+ }- newSession.duration =- new Date(newSession.ended_at).getTime() -- new Date(newSession.created_at).getTime();+ /**+ * Extend an in-flight session with a new event. Returns the updated+ * session AND the pair of CollapsingMergeTree rows that ClickHouse needs+ * to replace the previous row (sign=-1 cancels, sign=+1 inserts).+ */+ private extendSession(+ existing: IClickhouseSession,+ payload: IServiceCreateEventPayload+ ): {+ current: IClickhouseSession;+ chRows: [IClickhouseSession, IClickhouseSession];+ } {+ const oldRow: IClickhouseSession = { ...existing, sign: -1 };+ const current: IClickhouseSession = {+ ...existing,+ sign: 1,+ version: existing.version + 1,+ };++ // Out-of-order safety: treat the session window as [min(event ts),+ // max(event ts)]. Late-arriving events (offline mobile flush, retries)+ // can't drag ended_at backwards or push duration negative.+ const eventTimeMs = payload.createdAt.getTime();+ const startMs = fromClickhouseDate(current.created_at).getTime();+ const endMs = fromClickhouseDate(current.ended_at).getTime();+ const eventCh = toClickhouseDate(payload.createdAt);++ if (eventTimeMs >= endMs) {+ current.ended_at = eventCh;+ if (payload.path) current.exit_path = payload.path;+ if (payload.origin) current.exit_origin = payload.origin;+ }- const addedRevenue = event.name === 'revenue' ? (event.revenue ?? 0) : 0;- newSession.revenue = (newSession.revenue ?? 0) + addedRevenue;+ if (eventTimeMs < startMs) {+ current.created_at = eventCh;+ if (payload.path) current.entry_path = payload.path;+ if (payload.origin) current.entry_origin = payload.origin;+ } else {+ if (!current.entry_path && payload.path) current.entry_path = payload.path;+ if (!current.entry_origin && payload.origin)+ current.entry_origin = payload.origin;+ }- if (event.name === 'screen_view' && event.path) {- newSession.screen_view_count += 1;- } else {- newSession.event_count += 1;- }+ current.duration =+ fromClickhouseDate(current.ended_at).getTime() -+ fromClickhouseDate(current.created_at).getTime();- if (newSession.screen_view_count > 1) {- newSession.is_bounce = false;- }+ if (payload.name === 'revenue') {+ current.revenue = (current.revenue ?? 0) + (payload.revenue ?? 0);+ }- // If the profile_id is set and it's different from the device_id, we need to update the profile_id- if (event.profile_id && event.profile_id !== event.device_id) {- newSession.profile_id = event.profile_id;- }+ if (payload.name === 'screen_view' && payload.path) {+ current.screen_view_count += 1;+ } else {+ current.event_count += 1;+ }- if (event.groups) {- newSession.groups = [- ...new Set([...(newSession.groups ?? []), ...event.groups]),- ];- }+ if (current.screen_view_count > 1) {+ current.is_bounce = false;+ }- return [newSession, oldSession];+ if (payload.profileId && payload.profileId !== payload.deviceId) {+ current.profile_id = payload.profileId;}- return [- {- id: event.session_id,- is_bounce: true,- profile_id: event.profile_id,- project_id: event.project_id,- device_id: event.device_id,- groups: event.groups,- created_at: event.created_at,- ended_at: event.created_at,- event_count: event.name === 'screen_view' ? 0 : 1,- screen_view_count: event.name === 'screen_view' ? 1 : 0,- entry_path: event.path,- entry_origin: event.origin,- exit_path: event.path,- exit_origin: event.origin,- revenue: event.name === 'revenue' ? (event.revenue ?? 0) : 0,- referrer: event.referrer,- referrer_name: event.referrer_name,- referrer_type: event.referrer_type,- os: event.os,- os_version: event.os_version,- browser: event.browser,- browser_version: event.browser_version,- device: event.device,- brand: event.brand,- model: event.model,- country: event.country,- region: event.region,- city: event.city,- longitude: event.longitude ?? null,- latitude: event.latitude ?? null,- duration: event.duration,- utm_medium: event.properties?.['__query.utm_medium']- ? String(event.properties?.['__query.utm_medium'])- : '',- utm_source: event.properties?.['__query.utm_source']- ? String(event.properties?.['__query.utm_source'])- : '',- utm_campaign: event.properties?.['__query.utm_campaign']- ? String(event.properties?.['__query.utm_campaign'])- : '',- utm_content: event.properties?.['__query.utm_content']- ? String(event.properties?.['__query.utm_content'])- : '',- utm_term: event.properties?.['__query.utm_term']- ? String(event.properties?.['__query.utm_term'])- : '',- sign: 1,- version: 1,- },- ];+ if (payload.groups?.length) {+ current.groups = [+ ...new Set([...(current.groups ?? []), ...payload.groups]),+ ];+ }++ return { current, chRows: [current, oldRow] };}- protected getRedisListKey(): string {- return this.redisKey;+ /**+ * Open a brand-new session from this event's payload. Called for the first+ * event of a device AND on boundary (gap > 30min closing the old session).+ */+ private newSession(payload: IServiceCreateEventPayload): IClickhouseSession {+ const createdAt = toClickhouseDate(payload.createdAt);+ // For anonymous traffic the SDK doesn't send a profileId — fall back to+ // the deviceId so each anonymous device shows up as a unique visitor in+ // the dashboard. Matches what `createEvent` does for the events table.+ const profileId = payload.profileId || payload.deviceId;+ return {+ id: payload.sessionId,+ project_id: payload.projectId,+ device_id: payload.deviceId,+ profile_id: profileId,+ is_bounce: true,+ created_at: createdAt,+ ended_at: createdAt,+ event_count: payload.name === 'screen_view' ? 0 : 1,+ screen_view_count: payload.name === 'screen_view' ? 1 : 0,+ entry_path: payload.path ?? '',+ entry_origin: payload.origin ?? '',+ exit_path: payload.path ?? '',+ exit_origin: payload.origin ?? '',+ revenue: payload.name === 'revenue' ? (payload.revenue ?? 0) : 0,+ referrer: payload.referrer ?? '',+ referrer_name: payload.referrerName ?? '',+ referrer_type: payload.referrerType ?? '',+ os: payload.os ?? '',+ os_version: payload.osVersion ?? '',+ browser: payload.browser ?? '',+ browser_version: payload.browserVersion ?? '',+ device: payload.device ?? '',+ brand: payload.brand ?? '',+ model: payload.model ?? '',+ country: payload.country ?? '',+ region: payload.region ?? '',+ city: payload.city ?? '',+ longitude: payload.longitude ?? null,+ latitude: payload.latitude ?? null,+ duration: payload.duration ?? 0,+ utm_medium: pickUtm(payload, 'utm_medium'),+ utm_source: pickUtm(payload, 'utm_source'),+ utm_campaign: pickUtm(payload, 'utm_campaign'),+ utm_content: pickUtm(payload, 'utm_content'),+ utm_term: pickUtm(payload, 'utm_term'),+ groups: payload.groups ?? [],+ sign: 1,+ version: 1,+ };}- async add(event: IClickhouseEvent) {- if (!event.session_id) {- return;- }+ /**+ * Atomic write-back: session blob + wallclock ZSET + projects SET ++ * profile index + ClickHouse buffer rows + counter.+ *+ * No TTLs on the blob or profile index — they are removed exclusively by+ * `cleanup()` after `session_end` emission. This guarantees the reaper+ * can always find a session's state regardless of how long the worker+ * has been down.+ */+ private async persist(+ current: IClickhouseSession,+ chRows: IClickhouseSession[]+ ) {+ const projectId = current.project_id;+ const deviceId = current.device_id;+ const wallClockMs = Date.now();- if (['session_start', 'session_end'].includes(event.name)) {- return;- }+ const multi = this.redis.multi();+ multi.set(sessionKey(projectId, deviceId), JSON.stringify(current));+ multi.zadd(wallclockSetKey(projectId), wallClockMs.toString(), deviceId);+ multi.sadd(PROJECTS_SET_KEY, projectId);- return this.timeAdd(async () => {- try {- // Plural since we will delete the old session with sign column- const sessions = await this.getSession(event);- const [newSession] = sessions;-- const multi = this.redis.multi();- multi.set(- `session:${newSession.id}`,- JSON.stringify(newSession),- 'EX',- 60 * 60- );- if (newSession.profile_id) {- multi.set(- `session:${newSession.project_id}:${newSession.profile_id}`,- newSession.id,- 'EX',- 60 * 60- );- }- for (const session of sessions) {- multi.rpush(this.redisKey, JSON.stringify(session));- }- // Append LLEN at the end so we can read ground-truth queue length- // from the exec result without an extra round-trip.- multi.llen(this.redisKey);- const result = await multi.exec();+ if (current.profile_id && current.profile_id !== current.device_id) {+ multi.set(profileIndexKey(projectId, current.profile_id), deviceId);+ }- const llenIndex = (result?.length ?? 1) - 1;- const bufferLength = (result?.[llenIndex]?.[1] as number) ?? 0;+ for (const row of chRows) {+ multi.rpush(this.redisKey, JSON.stringify(row));+ }+ await multi.exec();- if (bufferLength >= this.batchSize) {- await this.tryFlush({ trigger: 'add' });- }- } catch (error) {- this.logger.error({ err: error }, 'Failed to add session');- }- });+ const bufferLength = await this.getBufferSize();+ if (bufferLength >= this.batchSize) {+ await this.tryFlush();+ }}/**@@ -349,65 +466,51 @@ export class SessionBuffer extends BaseBuffer {}async processBuffer() {- const lrangeStart = performance.now();const events = await this.redis.lrange(this.redisKey,0,this.batchSize - 1);- const lrangeMs = performance.now() - lrangeStart;-- if (events.length === 0) {- this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });- return;- }+ if (events.length === 0) return;const parsed: IClickhouseSession[] = [];- for (const raw of events) {- const session = getSafeJson<IClickhouseSession>(raw);- if (!session) {- continue;- }- parsed.push({- ...session,- duration: Math.max(0, session.duration || 0),- });+ for (const e of events) {+ const s = getSafeJson<IClickhouseSession>(e);+ if (!s) continue;+ // Freshly parsed object — safe to clamp in place, no clone needed.+ s.duration = Math.max(0, s.duration || 0);+ parsed.push(s);}- // A single high-activity session can produce many (sign=-1, sign=+1)- // pairs within one flush window — each event for that session adds a- // pair. The VersionedCollapsingMergeTree on `sessions` will collapse- // them at merge time, so the final state is correct either way. But- // inserting all the intermediate rows costs network bytes, gzip CPU,- // CH ingest work, and CH merge work.- //- // We can squash within the batch and only insert the boundary rows:- // the oldest sign=-1 (which cancels the previous CH state) and the- // newest sign=+1 (the final state). Same end result, fewer rows.+ // A single high-activity session produces many (sign=-1, sign=+1)+ // pairs within one flush window — every extend appends a pair. The+ // VersionedCollapsingMergeTree collapses them at merge time, so the+ // final state is correct either way, but inserting all the+ // intermediate rows costs network bytes, gzip CPU, and CH ingest ++ // merge work. Squash per-version to insert only the boundary rows.//// Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.const sessions = this.squashEnabled? this.squashSessionsByVersion(parsed): parsed;- const chStart = performance.now();- await this.parallelLimit(this.chunks(sessions, this.chunkSize), (chunk) =>- ch.insert({+ for (const chunk of this.chunks(sessions, this.chunkSize)) {+ await ch.insert({table: TABLE_NAMES.sessions,values: chunk,format: 'JSONEachRow',- clickhouse_settings: this.getClickhouseSettings(),- })- );- const chInsertMs = performance.now() - chStart;+ });+ }- const trimStart = performance.now();+ // Trim only after a successful insert — on failure the rows stay queued.+ // Don't swallow: let it propagate so the base flush records result:'error'+ // (and the next flush retries the still-queued rows).await this.redis.ltrim(this.redisKey, events.length, -1);- const trimMs = performance.now() - trimStart;- this.reportFlushStats({- rowsProcessed: events.length,- phases: { lrangeMs, chInsertMs, trimMs },- });+ this.logger.debug({ count: events.length }, 'Processed sessions');+ }++ protected getRedisListKey(): string {+ return this.redisKey;}}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex b31c5bafa..981c17f1d 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -4,7 +4,7 @@ import type { IChartEventFilter } from '@openpanel/validation';import { assocPath, last, mergeDeepRight, path, uniq } from 'ramda';import sqlstring from 'sqlstring';import { v4 as uuid } from 'uuid';-import { botBuffer, eventBuffer, sessionBuffer } from '../buffers';+import { botBuffer, eventBuffer } from '../buffers';import {ch,chQuery,@@ -355,6 +355,16 @@ export async function getEvents(return events.map(transformEvent);}+/**+ * Persist an event to ClickHouse (via the buffer) and upsert the profile+ * on session boundaries.+ *+ * Does NOT touch the session-row buffer. Callers producing non-session_start+ * / session_end events are responsible for calling `sessionBuffer.ingest()`+ * before this. `incoming-event.ts` is the only such caller today; everywhere+ * else (session_start, session_end) the session-row update is correctly a+ * no-op anyway.+ */export async function createEvent(payload: IServiceCreateEventPayload) {if (!payload.profileId && payload.deviceId) {payload.profileId = payload.deviceId;@@ -396,7 +406,9 @@ export async function createEvent(payload: IServiceCreateEventPayload) {groups: payload.groups ?? [],};- const promises = [sessionBuffer.add(event), eventBuffer.add(event)];+ eventBuffer.add(event);++ const promises: Promise<unknown>[] = [];if (payload.profileId) {const profile: IServiceUpsertProfile = {diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex 4c71c3f12..7a3321e47 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -50,9 +50,6 @@ export interface IClickhouseSession {utm_content: string;utm_term: string;revenue: number;- // CollapsingMergeTree marker: +1 = current state, -1 = cancel a prior +1.- // The session-buffer emits both +1 (new) and -1 (old) rows per update so- // CH can collapse intermediate states.sign: 1 | -1;version: number;// Dynamically added@@ -213,7 +210,7 @@ export async function getSessionList(options: GetSessionListOptions) {groupsExpr: 'groups',startDate,endDate,- }),+ }));}@@ -338,7 +335,7 @@ export async function getSessionsCount({groupsExpr: 'groups',startDate,endDate,- }),+ }));}@@ -481,7 +478,7 @@ export interface QuerySessionsInput {}export async function querySessionsCore(- input: QuerySessionsInput,+ input: QuerySessionsInput): Promise<IClickhouseSession[]> {const builder = clix(ch).select<IClickhouseSession>([])@@ -525,7 +522,10 @@ export async function querySessionsCore(builder.where('browser', '=', input.browser);}- const { startDate: start, endDate: end } = resolveDateRange(input.startDate, input.endDate);+ const { startDate: start, endDate: end } = resolveDateRange(+ input.startDate,+ input.endDate+ );builder.where('created_at', 'BETWEEN', [clix.datetime(start),diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 88398b9e7..9dfd096f4 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -1,5 +1,6 @@import { createHash } from 'node:crypto';import type {+ IClickhouseSession,IServiceCreateEventPayload,IServiceEvent,Prisma,@@ -76,6 +77,11 @@ export interface EventsQueuePayloadCreateEvent {export interface EventsQueuePayloadCreateSessionEnd {type: 'createSessionEnd';payload: IServiceCreateEventPayload;+ // Snapshot of the session at the moment the close was decided. Used as a+ // fallback when the live Redis blob has expired by the time the job runs,+ // and to detect post-enqueue extensions (so we don't close a session that+ // received more events in the meantime).+ snapshot: IClickhouseSession;}// TODO: Rename `EventsQueuePayloadCreateSessionEnd`@@ -138,6 +144,14 @@ export type CronQueuePayloadCohortRefresh = {type: 'cohortRefresh';payload: undefined;};+export type CronQueuePayloadSessionReaper = {+ type: 'sessionReaper';+ payload: undefined;+};+export type CronQueuePayloadSessionVacuum = {+ type: 'sessionVacuum';+ payload: undefined;+};export type CronQueuePayload =| CronQueuePayloadSalt| CronQueuePayloadFlushEvents@@ -151,7 +165,9 @@ export type CronQueuePayload =| CronQueuePayloadInsightsDaily| CronQueuePayloadOnboarding| CronQueuePayloadGscSync- | CronQueuePayloadCohortRefresh;+ | CronQueuePayloadCohortRefresh+ | CronQueuePayloadSessionReaper+ | CronQueuePayloadSessionVacuum;export type MiscQueuePayloadTrialEndingSoon = {type: 'trialEndingSoon';