OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
/**
* Session stress + drain-to-completion.
*
* Ramps out many short sessions (default 500, not all at once), then drives the
* reaper + buffer flushes until EVERYTHING has fully drained — every
* session_end emitted, Redis cleaned, the session buffer empty — and reconciles
* the ClickHouse counts. Exits 0 only when nothing is left open.
*
* Also instruments the run: per-request /track latency
* (P50/P95/P99, emit phase only), RSS/CPU of the api + worker processes (>=1Hz,
* peak + steady-state), Kafka consumer-group lag on the events topic (>=1Hz;
* peak, lag at emit-end, seconds-to-zero after emit stops), and every BullMQ
* queue depth + buffer pending count scraped off /metrics (>=1Hz; peak, depth
* at emit-end, whether it returned to zero). None of it changes what
* the run asserts — see `check` calls below.
*
* `saturation-sweep.ts` drives this script across a ladder of offered loads;
* see docs/BENCHMARK_HARNESS.md.
*
* Run (shrink the idle window; start the stack with the SAME value):
* SESSION_TIMEOUT_MS=4000 pnpm dev SESSION_TIMEOUT_MS=4000 pnpm --filter
* @openpanel/api e2e:sessions:stress
*
* Tunables (env): E2E_SESSIONS (500), E2E_CONCURRENCY (25),
* E2E_EVENTS_PER_SESSION (3), E2E_DRAIN_TIMEOUT_MS (120000),
* E2E_RECONCILE_TIMEOUT_MS (30000), E2E_RUNS (1) — run the whole harness
* E2E_RUNS times (as separate processes, so no state leaks between them) and
* print a median summary across runs at the end. E2E_NO_SAMPLING (0) — skip
* starting the process/lag monitors entirely (per-request latency still
* records; it's inherent to what's measured, not background polling). Control
* lever for isolating whether the >=1Hz sampling itself is dragging on
* throughput, e.g. `E2E_RUNS=3 E2E_NO_SAMPLING=1`. E2E_AUTH_MODE (bypass) —
* which ingest auth path every request takes: `bypass` | `cors` | `secret`; see
* AUTH_PATH_DESCRIPTION in lib.ts. The default keeps this script on the path
* the pre-existing suites were written against; `saturation-sweep.ts` defaults
* to `secret`. E2E_AUTH_WARM_REQUESTS (3) — authenticated requests sent and
* discarded before recording starts, so the scrypt verify behind
* VERIFY_CACHE_SECONDS is paid outside the measured window.
*/
import { spawn } from 'node:child_process';
import { formatLagSummary, LagMonitor, type LagSummary } from './lag-monitor';
import {
API_URL,
AUTH_MODE,
AUTH_PATH_DESCRIPTION,
AUTH_WARM_REQUESTS,
type AuthWarmResult,
beginLatencyRecording,
check,
checkCount,
chQuery,
countByName,
endLatencyRecording,
ensureFixtures,
formatLatencyStats,
getBlob,
IDLE_WAIT_MS,
type LatencyStats,
PROJECT_ID,
pollUntil,
preflight,
redis,
runId,
runPool,
SESSION_BUFFER_LIST,
SESSION_TIMEOUT_MS,
scenario,
screenView,
shutdown,
sleep,
summarize,
summarizeLatencies,
track,
triggerCron,
triggerReaper,
WORKER_URL,
wallclockKey,
warmAuthCache,
} from './lib';
import {
formatMetricsSummary,
MetricsMonitor,
type MetricsSummary,
} from './metrics-monitor';
import {
formatProcessSummary,
ProcessMonitor,
type ProcessSummary,
} from './process-monitor';
const SESSIONS = Number.parseInt(process.env.E2E_SESSIONS || '500', 10);
const CONCURRENCY = Number.parseInt(process.env.E2E_CONCURRENCY || '25', 10);
const EVENTS_PER_SESSION = Number.parseInt(
process.env.E2E_EVENTS_PER_SESSION || '3',
10
);
const DRAIN_TIMEOUT_MS = Number.parseInt(
process.env.E2E_DRAIN_TIMEOUT_MS || '120000',
10
);
// drain() exits on session_end + a clean Redis, which says nothing about the
// run's trailing regular events — those can still sit in the event buffer.
const RECONCILE_TIMEOUT_MS = Number.parseInt(
process.env.E2E_RECONCILE_TIMEOUT_MS || '30000',
10
);
const RECONCILE_INTERVAL_MS = 1500;
// Consecutive samples with no new rows before we call the buffer drained and
// let the assertion report whatever is actually there.
const RECONCILE_STABLE_SAMPLES = 3;
// The reconcile window opens this far before the run starts: a session_start
// is backdated 100ms from the event that opened it, and the first requests
// leave within milliseconds of the captured start.
const RECONCILE_WINDOW_LEAD_MS = 2000;
// Sampling runs at >=1Hz; 1000ms is the floor, not a target.
const SAMPLE_INTERVAL_MS = 1000;
const E2E_RUNS = Number.parseInt(process.env.E2E_RUNS || '1', 10);
// Repeat mode re-execs this same script as a child process per run, so no
// module-level state (redis/kafka connections, cached pids, latency sink)
// leaks between runs. This env var marks "I am one of those children" —
// print a machine-parseable result line instead of the human summary.
const IS_REPEAT_CHILD = process.env.E2E_STRESS_CHILD === '1';
const RESULT_MARKER = 'E2E_STRESS_RESULT ';
// Control lever: a run with this set skips the process/lag monitors so
// their >=1Hz polling can be ruled in or out as the cause of a throughput
// delta. Latency recording stays on regardless — it wraps the request the
// run is already making, not an added background poller.
const SAMPLING_ENABLED = process.env.E2E_NO_SAMPLING !== '1';
type Session = { sessionId: string; deviceId: string };
export interface RunResult {
ok: boolean;
failedChecks: number;
totalChecks: number;
/** The offered load this run was configured with, so a sweep can label its rungs. */
sessions: number;
eventsPerSession: number;
concurrency: number;
/** Which of the three ingest auth paths every measured request took. */
authMode: string;
authWarm: AuthWarmResult | null;
emitSeconds: number;
eventsPerSecond: number;
latency: LatencyStats;
api: ProcessSummary;
worker: ProcessSummary;
lag: LagSummary;
metrics: MetricsSummary;
samplingEnabled: boolean;
}
// Unique IP per session → unique device → unique session. Namespaced by runId
// so reruns and the correctness harness never collide.
const ipForSession = (i: number) =>
`100.${(runId >> 8) & 255}.${(i >> 8) & 255}.${i & 255}`;
// A realistic-ish mix: events are NOT all screen_views.
const CUSTOM_EVENTS = [
'sign_up',
'login',
'add_to_cart',
'purchase',
'search',
'button_click',
'video_play',
'share',
];
const WORDS = ['alpha', 'beta', 'gamma', 'sverige', 'katt', 'planet'];
const pick = <T>(arr: T[]): T => arr[Math.floor(Math.random() * arr.length)]!;
function randomProps(ip: string): Record<string, unknown> {
const props: Record<string, unknown> = { __ip: ip };
const n = Math.floor(Math.random() * 4);
for (let k = 0; k < n; k++) {
props[`prop_${pick(WORDS)}`] =
Math.random() < 0.5 ? pick(WORDS) : Math.floor(Math.random() * 1000);
}
return props;
}
// One event in a session: the entry is always a screen_view; the rest are a
// 50/50 mix of more screen_views and random custom events.
function sendSessionEvent(ip: string, i: number, e: number) {
if (e === 0 || Math.random() < 0.5) {
return screenView(ip, `/p${i}/${e}`, randomProps(ip));
}
return track(
{
type: 'track',
payload: { name: pick(CUSTOM_EVENTS), properties: randomProps(ip) },
},
ip
);
}
async function emit(): Promise<{
sessions: Session[];
latency: LatencyStats;
emitSeconds: number;
}> {
scenario(
`emit ${SESSIONS} sessions × ${EVENTS_PER_SESSION} events (concurrency ${CONCURRENCY})`
);
const sessions: Session[] = [];
let done = 0;
let errors = 0;
const started = Date.now();
const latencySamples = beginLatencyRecording();
await runPool(
Array.from({ length: SESSIONS }, (_, i) => i),
CONCURRENCY,
async (i) => {
const ip = ipForSession(i);
try {
const first = await sendSessionEvent(ip, i, 0);
for (let e = 1; e < EVENTS_PER_SESSION; e++) {
await sendSessionEvent(ip, i, e);
}
sessions.push({ sessionId: first.sessionId, deviceId: first.deviceId });
} catch (error) {
errors++;
if (errors <= 3) {
console.warn(` session ${i} failed: ${(error as Error).message}`);
}
}
done++;
if (done % 100 === 0 || done === SESSIONS) {
console.log(` …emitted ${done}/${SESSIONS}`);
}
}
);
endLatencyRecording();
const uniq = new Set(sessions.map((s) => s.sessionId)).size;
check(
'all sessions sent without HTTP errors',
errors === 0,
`${errors} errors`
);
check(
'each session got a distinct session id',
uniq === SESSIONS,
`${uniq} unique / ${sessions.length} ok / ${SESSIONS} sent`
);
const emitSeconds = (Date.now() - started) / 1000;
const latency = summarizeLatencies(latencySamples);
console.log(` emit took ${emitSeconds.toFixed(1)}s`);
console.log(` ${formatLatencyStats(latency)}`);
return { sessions, latency, emitSeconds };
}
async function settle(sessions: Session[]) {
scenario('settle: all sessions opened in ClickHouse');
const ids = sessions.map((s) => s.sessionId);
const starts = await pollUntil(
async () => {
const c = await countByName(ids, 'session_start');
console.log(` …session_start ${c}/${ids.length}`);
return c >= ids.length ? c : null;
},
{ timeoutMs: 60_000, intervalMs: 2000 }
);
check(
'clickhouse: one session_start per session',
starts === ids.length,
`got ${starts}`
);
}
async function drain(sessions: Session[]) {
scenario('drain: reap + flush until nothing remains');
console.log(` …waiting ${IDLE_WAIT_MS}ms for all sessions to idle out`);
await sleep(IDLE_WAIT_MS);
const ids = sessions.map((s) => s.sessionId);
const target = ids.length;
const deadline = Date.now() + DRAIN_TIMEOUT_MS;
let ends = 0;
let remaining = Number.POSITIVE_INFINITY;
while (Date.now() < deadline) {
await triggerReaper(); // enqueue session_end for idle sessions
await sleep(1000); // let the worker process the close jobs
await triggerCron('flushEvents'); // push session_start/end + events → CH
await triggerCron('flushSessions'); // push sessions table rows → CH
ends = await countByName(ids, 'session_end');
remaining = await redis.zcard(wallclockKey);
console.log(
` …session_end ${ends}/${target}, wallclock remaining ${remaining}`
);
if (ends >= target && remaining === 0) {
break;
}
await sleep(1500);
}
check(
'clickhouse: one session_end per session',
ends === target,
`got ${ends}`
);
check(
'redis: wallclock index fully drained',
remaining === 0,
`${remaining} left`
);
const bufLen = await redis.llen(SESSION_BUFFER_LIST);
check(
'redis: session buffer drained to ClickHouse',
bufLen === 0,
`${bufLen} rows pending`
);
// Spot-check a sample of devices: no session blob should linger after close.
const sample = sessions.filter(
(_, k) => k % Math.ceil(sessions.length / 20) === 0
);
let leaked = 0;
for (const s of sample) {
if (await getBlob(s.deviceId)) {
leaked++;
}
}
check(
'redis: no session blobs leaked (sampled)',
leaked === 0,
`${leaked}/${sample.length} leaked`
);
}
/**
* Count this run's rows by event name, scoped to the run's DEVICES rather than
* to the session ids `/track` echoed back.
*
* Why the device: until the worker has persisted the session blob, the API
* answers from a session id that is deterministic per SESSION_TIMEOUT_MS-wide
* time bucket. At the harness's compressed 4s window a bucket boundary falls
* inside the emit ramp, so an in-flight
* session's next event is told a different id — the row still lands, under a
* session id the harness was never given. The device is the stable identity
* for "what this run sent"; `since` keeps earlier runs' rows out, because a
* device id repeats across runs whenever the ip/ua/salt triple does.
*/
async function countEventsByName(deviceIds: string[], since: Date) {
const inList = deviceIds.map((id) => `'${id}'`).join(',');
const from = new Date(since.getTime() - RECONCILE_WINDOW_LEAD_MS)
.toISOString()
.replace('T', ' ')
.replace('Z', '');
const rows = await chQuery<{ name: string; c: string }>(
`SELECT name, count() AS c FROM events WHERE project_id = '${PROJECT_ID}' AND device_id IN (${inList}) AND created_at >= toDateTime64('${from}', 3) GROUP BY name ORDER BY c DESC`
);
const byName = new Map(rows.map((r) => [r.name, Number(r.c)]));
const total = [...byName.values()].reduce((a, b) => a + b, 0);
return { rows, byName, total };
}
async function reconcile(sessions: Session[], since: Date) {
scenario('reconcile: ClickHouse event counts');
const devices = sessions.map((s) => s.deviceId);
const n = sessions.length;
const expectedTotal = n * (EVENTS_PER_SESSION + 2);
let counts = await countEventsByName(devices, since);
const deadline = Date.now() + RECONCILE_TIMEOUT_MS;
let stableSamples = 0;
while (
counts.total < expectedTotal &&
stableSamples < RECONCILE_STABLE_SAMPLES &&
Date.now() < deadline
) {
console.log(` …events ${counts.total}/${expectedTotal}, flushing`);
await triggerCron('flushEvents');
await sleep(RECONCILE_INTERVAL_MS);
const next = await countEventsByName(devices, since);
stableSamples = next.total > counts.total ? 0 : stableSamples + 1;
counts = next;
}
const { rows, byName, total } = counts;
console.log(
` event breakdown: ${rows.map((r) => `${r.name}=${r.c}`).join(' ')}`
);
const starts = byName.get('session_start') ?? 0;
const ends = byName.get('session_end') ?? 0;
// Each session = N events we sent + the synthetic session_start + session_end.
check('session_start == sessions', starts === n, `${starts} vs ${n}`);
check('session_end == sessions', ends === n, `${ends} vs ${n}`);
check(
'total events == sessions × (events + start + end)',
total === expectedTotal,
`${total} vs ${expectedTotal}`
);
}
const portFromUrl = (url: string) => Number(new URL(url).port);
/**
* Pay the auth cost BEFORE recording starts, so the scrypt behind
* `VERIFY_CACHE_SECONDS` can never land inside the measured window and be read
* as a knee. Discarded on purpose: the warm requests are not in the latency
* series, not in the reconcile counts (they use their own ip), and the
* sessions they open are reaped by the same drain loop as the rest.
*/
async function warmAuth(): Promise<AuthWarmResult | null> {
if (AUTH_WARM_REQUESTS <= 0) {
return null;
}
const warm = await warmAuthCache(AUTH_MODE, AUTH_WARM_REQUESTS);
console.log(
` auth cache warmed: ${warm.requests} discarded requests, ` +
`${warm.latenciesMs.map((ms) => ms.toFixed(1)).join('/')}ms (max ${warm.maxMs?.toFixed(1)}ms)`
);
return warm;
}
async function runOnce(): Promise<RunResult> {
console.log(
`Session STRESS — api=${API_URL} worker=${WORKER_URL} timeout=${SESSION_TIMEOUT_MS}ms ` +
`sessions=${SESSIONS} events=${EVENTS_PER_SESSION} concurrency=${CONCURRENCY}` +
(SAMPLING_ENABLED ? '' : ' [sampling disabled — control run]')
);
console.log(` auth path: ${AUTH_PATH_DESCRIPTION[AUTH_MODE]}`);
await preflight();
await ensureFixtures();
const authWarm = await warmAuth();
const apiMonitor = new ProcessMonitor('api', portFromUrl(API_URL));
const workerMonitor = new ProcessMonitor('worker', portFromUrl(WORKER_URL));
const lagMonitor = new LagMonitor();
const metricsMonitor = new MetricsMonitor(API_URL);
if (SAMPLING_ENABLED) {
await Promise.all([
apiMonitor.start(SAMPLE_INTERVAL_MS),
workerMonitor.start(SAMPLE_INTERVAL_MS),
lagMonitor.start(SAMPLE_INTERVAL_MS),
metricsMonitor.start(SAMPLE_INTERVAL_MS),
]);
}
try {
// Captured before the first event so the reconcile window covers every row
// of this run, including session_start (backdated 100ms).
const runStartedAt = new Date();
const { sessions, latency, emitSeconds } = await emit();
const emitEndedAt = Date.now();
await settle(sessions);
await drain(sessions);
await reconcile(sessions, runStartedAt);
apiMonitor.stop();
workerMonitor.stop();
metricsMonitor.stop();
await lagMonitor.stop();
const api = apiMonitor.summary();
const worker = workerMonitor.summary();
const lag = lagMonitor.summarize(emitEndedAt);
const metrics = metricsMonitor.summarize(emitEndedAt);
scenario('instrumentation summary');
console.log(` auth path: ${AUTH_PATH_DESCRIPTION[AUTH_MODE]}`);
console.log(` ${formatProcessSummary(api)}`);
console.log(` ${formatProcessSummary(worker)}`);
console.log(` ${formatLagSummary(lag)}`);
for (const line of formatMetricsSummary(metrics)) {
console.log(` ${line}`);
}
const failedChecks = summarize();
return {
ok: failedChecks === 0,
failedChecks,
totalChecks: checkCount(),
sessions: SESSIONS,
eventsPerSession: EVENTS_PER_SESSION,
concurrency: CONCURRENCY,
authMode: AUTH_MODE,
authWarm,
emitSeconds,
eventsPerSecond: (SESSIONS * EVENTS_PER_SESSION) / emitSeconds,
latency,
api,
worker,
lag,
metrics,
samplingEnabled: SAMPLING_ENABLED,
};
} finally {
apiMonitor.stop();
workerMonitor.stop();
metricsMonitor.stop();
await lagMonitor.stop();
}
}
const median = (values: number[]): number | null => {
const finite = values.filter((v) => Number.isFinite(v));
if (finite.length === 0) {
return null;
}
const sorted = [...finite].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0
? ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2
: (sorted[mid] ?? 0);
};
const fmt = (n: number | null, unit = ''): string =>
n === null ? 'n/a' : `${n.toFixed(1)}${unit}`;
const kbToMb = (kb: number | null): number | null =>
kb === null ? null : kb / 1024;
function printMedianSummary(results: RunResult[], attempted: number) {
const suffix = SAMPLING_ENABLED ? '' : ' [sampling disabled — control run]';
console.log(`\n${'═'.repeat(60)}`);
console.log(
`Median summary across ${results.length}/${attempted} completed runs${suffix}`
);
console.log('═'.repeat(60));
if (results.length === 0) {
console.log(' no run produced a result');
return;
}
// Median of a per-run metric across all completed runs.
const m = (f: (r: RunResult) => number | null) =>
median(results.map(f).filter((v): v is number => v !== null));
console.log(
` emit: ${fmt(
m((r) => r.emitSeconds),
's'
)} (${fmt(
m((r) => r.eventsPerSecond),
' ev/s'
)})`
);
console.log(
` /track latency: P50=${fmt(
m((r) => r.latency.p50),
'ms'
)} P95=${fmt(
m((r) => r.latency.p95),
'ms'
)} P99=${fmt(
m((r) => r.latency.p99),
'ms'
)}`
);
console.log(
` api RSS/CPU: peak=${fmt(kbToMb(m((r) => r.api.peakRssKb)), 'MB')} steady=${fmt(kbToMb(m((r) => r.api.steadyRssKb)), 'MB')} / ` +
`peak=${fmt(
m((r) => r.api.peakCpuPct),
'%'
)} steady=${fmt(
m((r) => r.api.steadyCpuPct),
'%'
)}`
);
console.log(
` worker RSS/CPU: peak=${fmt(kbToMb(m((r) => r.worker.peakRssKb)), 'MB')} steady=${fmt(kbToMb(m((r) => r.worker.steadyRssKb)), 'MB')} / ` +
`peak=${fmt(
m((r) => r.worker.peakCpuPct),
'%'
)} steady=${fmt(
m((r) => r.worker.steadyCpuPct),
'%'
)}`
);
console.log(
` kafka lag: peak=${fmt(m((r) => r.lag.peakLag))} at-emit-end=${fmt(m((r) => r.lag.lagAtEmitEnd))} ` +
`seconds-to-zero=${fmt(
m((r) => r.lag.secondsToZeroAfterEmitEnd),
's'
)}`
);
const greenRuns = results.filter((r) => r.ok).length;
console.log(` green runs: ${greenRuns}/${attempted}`);
}
function runChild(index: number, total: number): Promise<RunResult> {
console.log(
`\n${'='.repeat(60)}\nRun ${index + 1}/${total}\n${'='.repeat(60)}`
);
return new Promise<RunResult>((resolve, reject) => {
// Re-run the same package.json script rather than replaying argv: argv[1]
// here is a bare.ts path that only resolves because the *current* process
// already has jiti's loader hook registered in-process — a fresh node/bun
// invocation needs the same wrapper the user ran ("pnpm run <script>") to
// get that hook again. This also means the repeat mode keeps working
// unchanged once the toolchain moves to Bun: whatever `e2e:sessions:stress`
// runs then is what gets re-exec'd.
const child = spawn('pnpm', ['run', 'e2e:sessions:stress'], {
cwd: process.cwd(),
env: { ...process.env, E2E_RUNS: '1', E2E_STRESS_CHILD: '1' },
stdio: ['ignore', 'pipe', 'inherit'],
});
let buffered = '';
child.stdout.on('data', (chunk: Buffer) => {
const text = chunk.toString();
process.stdout.write(text);
buffered += text;
});
child.on('error', reject);
child.on('close', (code) => {
const line = buffered
.split('\n')
.find((l) => l.startsWith(RESULT_MARKER));
if (!line) {
reject(
new Error(
`run ${index + 1} produced no parseable result (exit ${code})`
)
);
return;
}
resolve(JSON.parse(line.slice(RESULT_MARKER.length)) as RunResult);
});
});
}
async function runRepeated(n: number) {
const results: RunResult[] = [];
for (let i = 0; i < n; i++) {
try {
results.push(await runChild(i, n));
} catch (error) {
console.error(
`\nRun ${i + 1}/${n} did not complete: ${(error as Error).message}`
);
}
}
printMedianSummary(results, n);
const allGreen = results.length === n && results.every((r) => r.ok);
await shutdown(allGreen ? 0 : 1);
}
// process.exit() right after a plain console.log can truncate a piped
// stdout (Node flushes it async) — wait for the write's own callback so the
// parent always sees the full marker line before we exit.
function writeResultMarker(result: RunResult): Promise<void> {
return new Promise((resolve) => {
process.stdout.write(`${RESULT_MARKER}${JSON.stringify(result)}\n`, () =>
resolve()
);
});
}
async function main() {
if (E2E_RUNS > 1 && !IS_REPEAT_CHILD) {
await runRepeated(E2E_RUNS);
return;
}
const result = await runOnce();
if (IS_REPEAT_CHILD) {
await writeResultMarker(result);
}
await shutdown(result.failedChecks);
}
main().catch(async (error) => {
console.error('\nFATAL:', error);
await shutdown(1);
});