OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(ingest): a buffer flush keeps its lock for as long as it runs (#570)
Carl-Gerhard Lindesvärd4c07fab1a662
diff --git a/apps/api/e2e/README.md b/apps/api/e2e/README.mdindex 0d5b6b4f4..19047a62d 100644--- a/apps/api/e2e/README.md+++ b/apps/api/e2e/README.md@@ -15,6 +15,10 @@ over HTTP and asserting state in **both ClickHouse and Redis**:rung, and reports the max sustainable ingest throughput and where the knee is.Full knob list, the knee definition and what the numbers do NOT cover:**`docs/BENCHMARK_HARNESS.md`**.+- `ingest-load.ts` (`e2e:ingest-load`) — **bulk throughput**: N devices × M+ events through `/track`, timed from the first request until every business+ event is in ClickHouse and every buffer is empty, with the Redis command and+ CPU cost of the run. See "Ingest load" below.## What it covers@@ -67,6 +71,38 @@ endpoint, so it never waits for the 5-minute reaper cron.- The harness and the stack **must share the same `SESSION_TIMEOUT_MS`** — theharness derives its idle waits from it.+## Ingest load (`ingest-load.ts`)++The run behind "100,000 events in 27 s" in #560, reproducible against any+running api + worker pair (self-hosted installs included):++```bash+# From apps/api. API_URL, REDIS_URL, CLICKHOUSE_URL and the Kafka settings come+# from the environment; the write client from .seed.json or --client/--secret.+bun run e2e:ingest-load -- --devices 5000 --per-device 20 --concurrency 100+```++Knobs: `--devices`, `--per-device`, `--concurrency`, `--properties N` (custom+properties per event, the last one an array), `--identified` (a `profileId`+per device), `--project`, `--client`/`--secret`, `--label`, `--out`.++It refuses to start unless Kafka lag is zero and all three buffers are empty,+then prints one line per second (`sent`, Kafka lag, buffer lengths, business+events in ClickHouse) and ends with:++- the marks: `sendDone`, `kafkaDrained`, `eventsInClickhouse` (business events+ only, `session_start`/`session_end` excluded), `buffersDrained`;+- the Redis `INFO commandstats` delta over the run, as calls per event and+ share of Redis time, and the `INFO cpu` delta as CPU milliseconds per event+ (main thread and snapshot forks separately) — a number that compares across+ machines where "% of a core" does not;+- a verification: event rows and unique ids, `session_start` rows, and+ sessions whose signed rows do not net to one.++The timer starts at the first request and excludes process startup and the+consumer-group join. Peak buffer lengths are reported so a run that only looks+fast because a buffer was still filling is visible as such.+## P1 baseline — measured throughput (2026-09-02)`BASE-002` never produced a V1 stress number (both databases were unusable ondiff --git a/apps/api/e2e/ingest-load.ts b/apps/api/e2e/ingest-load.tsnew file mode 100644index 000000000..76332e453--- /dev/null+++ b/apps/api/e2e/ingest-load.ts@@ -0,0 +1,469 @@+/**+ * Ingest throughput harness: N devices × M events through `POST /track`+ * (API → Kafka → worker → Redis buffers → ClickHouse), timed from the first+ * request until every business event is in ClickHouse and every buffer is+ * empty. The run that produced the "100,000 events in 27 s" figure in #560+ * is `--devices 5000 --per-device 20 --concurrency 100`.+ *+ * Besides the wall-clock marks it reports what the run cost Redis: the delta+ * of `INFO commandstats` and `INFO cpu` over the run, so Redis CPU can be+ * attributed to commands and compared across machines as CPU-seconds per+ * event rather than as a percentage of a core.+ *+ * Refuses to start unless the pipeline is idle (no Kafka lag, every partition+ * committed, all three buffers empty), so the marks measure this run alone.+ *+ * Usage (from apps/api, against a running api + worker):+ * dotenv -e ../../.env -- bun e2e/ingest-load.ts [options]+ * --devices 5000 --per-device 20 --concurrency 100+ * --properties 2 custom properties per event (one of them an array)+ * --identified send a profileId per device (identified traffic)+ * --project acme-shop project id, for the ClickHouse counts+ * --client / --secret write client; defaults to the seeded project's+ * --label run prefix for the output files in --out+ * --out /tmp/openpanel-ingest-load+ * Env: API_URL (or E2E_API_URL), REDIS_URL, CLICKHOUSE_URL, and the Kafka+ * topic and group from the api's own config loader.+ */++import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';+import { resolve } from 'node:path';+import { parseArgs } from 'node:util';+import { createKafkaAdmin, sampleConsumerGroupLag } from '@openpanel/core';+import { getRedisCache } from '@openpanel/redis';+import { loadConfig } from '../src/config/env';++const SAMPLE_INTERVAL_MS = 1000;+const DEFAULT_TIMEOUT_S = 1800;+const ARRAY_PROPERTY_LENGTH = 3;+const OCTET_RANGE = 256;+const TOP_COMMANDS = 14;+/** A reset keep-alive socket is retried like an SDK would; a 4xx/5xx is not. */+const SEND_ATTEMPTS = 3;+const BUFFER_KEYS = {+ event: 'event_buffer:queue',+ session: 'session-buffer',+ profile: 'profile-buffer',+} as const;+const USER_AGENT =+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/128.0 Safari/537.36';++const { values } = parseArgs({+ options: {+ devices: { type: 'string', default: '5000' },+ 'per-device': { type: 'string', default: '20' },+ concurrency: { type: 'string', default: '100' },+ properties: { type: 'string', default: '2' },+ identified: { type: 'boolean', default: false },+ project: { type: 'string', default: 'acme-shop' },+ client: { type: 'string' },+ secret: { type: 'string' },+ label: { type: 'string', default: 'run' },+ out: { type: 'string', default: '/tmp/openpanel-ingest-load' },+ timeout: { type: 'string', default: String(DEFAULT_TIMEOUT_S) },+ },+});++function integerOption(name: string, raw: string, min: number): number {+ const value = Number(raw);+ if (!Number.isInteger(value) || value < min) {+ throw new Error(`--${name} must be an integer >= ${min}, got "${raw}"`);+ }+ return value;+}++const devices = integerOption('devices', values.devices, 1);+const perDevice = integerOption('per-device', values['per-device'], 1);+const concurrency = integerOption('concurrency', values.concurrency, 1);+const propertyCount = integerOption('properties', values.properties, 0);+const timeoutS = integerOption('timeout', values.timeout, 1);+const total = devices * perDevice;+const projectId = values.project;+const runId = `${values.label}_${Date.now().toString(36)}`;+const outPrefix = resolve(values.out, runId);+mkdirSync(values.out, { recursive: true });++const apiUrl = process.env.E2E_API_URL ?? process.env.API_URL;+if (!apiUrl) {+ throw new Error('Set API_URL (or E2E_API_URL) to the api you are loading.');+}+const clickhouseUrl = process.env.CLICKHOUSE_URL;+if (!clickhouseUrl) {+ throw new Error('Set CLICKHOUSE_URL.');+}++interface SeedManifest {+ projects: { id: string; clientId: string; clientSecret: string }[];+}++function resolveClient(): { id: string; secret: string } {+ if (values.client && values.secret) {+ return { id: values.client, secret: values.secret };+ }+ const manifestPath = resolve(import.meta.dir, '../../../.seed.json');+ if (!existsSync(manifestPath)) {+ throw new Error(+ 'No --client/--secret and no .seed.json: run `bun run seed` or pass a write client.'+ );+ }+ const manifest = JSON.parse(+ readFileSync(manifestPath, 'utf8')+ ) as SeedManifest;+ const project = manifest.projects.find((p) => p.id === projectId);+ if (!project) {+ throw new Error(`Project ${projectId} is not in .seed.json.`);+ }+ return { id: project.clientId, secret: project.clientSecret };+}++const client = resolveClient();+const kafkaConfig = loadConfig().core;+const admin = createKafkaAdmin(kafkaConfig);+const redis = getRedisCache();++const log = (line: string) => {+ console.log(line);+ appendFileSync(`${outPrefix}.samples`, `${line}\n`);+};++async function clickhouse(query: string): Promise<string> {+ const url = new URL(clickhouseUrl as string);+ const database = url.pathname.slice(1);+ url.pathname = '/';+ if (database) {+ url.searchParams.set('database', database);+ }+ const response = await fetch(url, { method: 'POST', body: query });+ const text = await response.text();+ if (!response.ok) {+ throw new Error(text);+ }+ return text.trim();+}++const runFilter = `project_id = '${projectId}' AND properties['run'] = '${runId}'`;+const businessFilter = `${runFilter} AND name NOT IN ('session_start', 'session_end')`;++const countBusinessEvents = async () =>+ Number(+ await clickhouse(`SELECT count() FROM events WHERE ${businessFilter}`)+ );++async function kafkaState() {+ const [lag, groups] = await Promise.all([+ sampleConsumerGroupLag(+ admin,+ kafkaConfig.kafka.eventsTopic,+ kafkaConfig.kafka.consumerGroup+ ),+ admin.describeGroups([kafkaConfig.kafka.consumerGroup]),+ ]);+ const group = groups.groups[0];+ return {+ lag: lag.totalLag,+ state: group?.state ?? 'Unknown',+ members: group?.members.length ?? 0,+ };+}++async function bufferSizes() {+ const [event, session, profile] = await Promise.all([+ redis.llen(BUFFER_KEYS.event),+ redis.llen(BUFFER_KEYS.session),+ redis.llen(BUFFER_KEYS.profile),+ ]);+ return { event, session, profile };+}++type CommandStats = Map<string, { calls: number; usec: number }>;++async function redisCommandStats(): Promise<CommandStats> {+ const stats: CommandStats = new Map();+ const info = await redis.info('commandstats');+ for (const line of info.split('\n')) {+ const match = line.match(/^cmdstat_([^:]+):calls=(\d+),usec=(\d+)/);+ if (match) {+ stats.set(match[1] as string, {+ calls: Number(match[2]),+ usec: Number(match[3]),+ });+ }+ }+ return stats;+}++/** Main thread and the snapshot fork separately: RDB saves are the fork. */+async function redisCpuSeconds(): Promise<{ main: number; children: number }> {+ const info = await redis.info('cpu');+ const read = (key: string) =>+ Number(info.match(new RegExp(`${key}:([\\d.]+)`))?.[1] ?? 0);+ return {+ main: read('used_cpu_sys') + read('used_cpu_user'),+ children: read('used_cpu_sys_children') + read('used_cpu_user_children'),+ };+}++function reportRedisCost(before: CommandStats, after: CommandStats) {+ const rows: { command: string; calls: number; usec: number }[] = [];+ for (const [command, stat] of after) {+ const previous = before.get(command) ?? { calls: 0, usec: 0 };+ const calls = stat.calls - previous.calls;+ const usec = stat.usec - previous.usec;+ if (calls > 0) {+ rows.push({ command, calls, usec });+ }+ }+ rows.sort((a, b) => b.usec - a.usec);+ const totalUsec = rows.reduce((sum, row) => sum + row.usec, 0);+ log('Redis commands over the run (server-wide), by time spent:');+ log(+ ' command calls calls/event usec/call share of Redis time'+ );+ for (const row of rows.slice(0, TOP_COMMANDS)) {+ log(+ ` ${row.command.padEnd(16)} ${String(row.calls).padStart(8)} ${(row.calls / total).toFixed(2).padStart(13)} ${(row.usec / row.calls).toFixed(1).padStart(11)} ${((100 * row.usec) / totalUsec).toFixed(1).padStart(10)}%`+ );+ }+ log(+ ` total command time ${(totalUsec / 1e6).toFixed(1)} s = ${((1000 * totalUsec) / 1e6 / total).toFixed(3)} ms per business event`+ );+}++/** A distinct public-looking IP per device, so each gets its own device id. */+function ipFor(device: number) {+ const octets = [+ Math.floor(device / OCTET_RANGE ** 2) % OCTET_RANGE,+ Math.floor(device / OCTET_RANGE) % OCTET_RANGE,+ device % OCTET_RANGE,+ ];+ return `81.${octets.join('.')}`;+}++function customProperties(device: number, step: number) {+ const properties: Record<string, unknown> = {};+ for (let i = 1; i <= propertyCount; i++) {+ properties[`prop_${i}`] =+ i === propertyCount+ ? Array.from(+ { length: ARRAY_PROPERTY_LENGTH },+ (_, j) => `tag_${(step + j) % 7}`+ )+ : `v${(device * 31 + step * i) % 1000}`;+ }+ return properties;+}++async function assertIdle() {+ const [kafka, buffers] = await Promise.all([kafkaState(), bufferSizes()]);+ log(+ `preflight run=${runId} kafka=${JSON.stringify(kafka)} buffers=${JSON.stringify(buffers)}`+ );+ const idle =+ kafka.state === 'Stable' &&+ kafka.members > 0 &&+ kafka.lag === 0 &&+ buffers.event === 0 &&+ buffers.session === 0 &&+ buffers.profile === 0;+ if (!idle) {+ log('PREFLIGHT FAILED: pipeline is not idle, or no worker is consuming');+ process.exit(2);+ }+}++await assertIdle();++// Round-robin: event k of every device, then event k+1, so every session is+// open for the whole run and the worker sees the realistic interleaving.+const jobs: { device: number; step: number }[] = [];+for (let step = 0; step < perDevice; step++) {+ for (let device = 0; device < devices; device++) {+ jobs.push({ device, step });+ }+}++let accepted = 0;+let failed = 0;+let retried = 0;+let nextJob = 0;++async function send(job: { device: number; step: number }): Promise<Response> {+ const body = {+ type: 'track',+ payload: {+ name: job.step % 2 === 0 ? 'screen_view' : `load_${values.label}`,+ properties: {+ __path: `https://shop.example/p/${job.step}`,+ run: runId,+ ...customProperties(job.device, job.step),+ },+ ...(values.identified ? { profileId: `user_${job.device}` } : {}),+ },+ };+ let lastError: unknown;+ for (let attempt = 1; attempt <= SEND_ATTEMPTS; attempt++) {+ try {+ return await fetch(`${apiUrl}/track`, {+ method: 'POST',+ headers: {+ 'content-type': 'application/json',+ 'openpanel-client-id': client.id,+ 'openpanel-client-secret': client.secret,+ // The run id makes every run's device ids its own, so two runs never+ // extend the same session.+ 'user-agent': `${USER_AGENT} run/${runId}`,+ 'x-client-ip': ipFor(job.device + 1),+ },+ body: JSON.stringify(body),+ });+ } catch (error) {+ lastError = error;+ retried++;+ if (retried <= 5) {+ log(`retrying after ${(error as Error).message}`);+ }+ }+ }+ throw lastError;+}++async function sender() {+ while (nextJob < jobs.length) {+ const job = jobs[nextJob++] as { device: number; step: number };+ const response = await send(job);+ await response.arrayBuffer();+ if (response.ok) {+ accepted++;+ } else {+ failed++;+ if (failed <= 5) {+ log(`HTTP ${response.status}`);+ }+ }+ }+}++const statsBefore = await redisCommandStats();+const cpuBefore = await redisCpuSeconds();+const startedAt = performance.now();+const elapsed = () => (performance.now() - startedAt) / 1000;+const marks = {+ sendDone: 0,+ kafkaDrained: 0,+ eventsInClickhouse: 0,+ buffersDrained: 0,+};+const peaks = {+ kafkaLag: 0,+ eventBuffer: 0,+ sessionBuffer: 0,+ profileBuffer: 0,+};++/** Resolves true once everything landed, false on timeout. */+const monitor = (async (): Promise<boolean> => {+ while (true) {+ const [buffers, kafka, inClickhouse] = await Promise.all([+ bufferSizes(),+ kafkaState(),+ countBusinessEvents(),+ ]);+ const t = elapsed();+ peaks.kafkaLag = Math.max(peaks.kafkaLag, kafka.lag);+ peaks.eventBuffer = Math.max(peaks.eventBuffer, buffers.event);+ peaks.sessionBuffer = Math.max(peaks.sessionBuffer, buffers.session);+ peaks.profileBuffer = Math.max(peaks.profileBuffer, buffers.profile);+ log(+ `${t.toFixed(1)}s sent=${accepted} kafkaLag=${kafka.lag} evBuf=${buffers.event} sessBuf=${buffers.session} profBuf=${buffers.profile} chEvents=${inClickhouse}`+ );+ if (marks.sendDone) {+ if (!marks.kafkaDrained && kafka.lag === 0) {+ marks.kafkaDrained = t;+ }+ // A rejected request wrote no event, so the target is what was accepted.+ if (!marks.eventsInClickhouse && inClickhouse >= accepted) {+ marks.eventsInClickhouse = t;+ }+ const drained =+ buffers.event === 0 && buffers.session === 0 && buffers.profile === 0;+ if (marks.eventsInClickhouse && marks.kafkaDrained && drained) {+ marks.buffersDrained = t;+ return true;+ }+ }+ if (t > timeoutS) {+ log('TIMEOUT');+ return false;+ }+ await Bun.sleep(SAMPLE_INTERVAL_MS);+ }+})();++await Promise.all(Array.from({ length: concurrency }, sender));+marks.sendDone = elapsed();+log(+ `send done at ${marks.sendDone.toFixed(1)}s (${(accepted / marks.sendDone).toFixed(0)} accepted requests/s)`+);+const completed = await monitor;+// On a timeout the rates are over the time the run was given.+const runSeconds = completed ? marks.buffersDrained : elapsed();++const cpuAfter = await redisCpuSeconds();+reportRedisCost(statsBefore, await redisCommandStats());+const redisCpu = cpuAfter.main - cpuBefore.main;+const redisForkCpu = cpuAfter.children - cpuBefore.children;+log(+ `Redis CPU over the run: ${redisCpu.toFixed(1)} s = ${((1000 * redisCpu) / total).toFixed(3)} ms per business event (${((100 * redisCpu) / runSeconds).toFixed(0)}% of one core on average), plus ${redisForkCpu.toFixed(1)} s in snapshot forks`+);++const [eventCounts, sessionStarts, sessionCounts] = await Promise.all([+ clickhouse(+ `SELECT count(), uniqExact(id) FROM events WHERE ${businessFilter} FORMAT TSV`+ ),+ clickhouse(+ `SELECT count() FROM events WHERE ${runFilter} AND name = 'session_start' FORMAT TSV`+ ),+ clickhouse(+ `SELECT count(), countIf(net != 1) FROM (+ SELECT id, sum(sign) AS net FROM sessions+ WHERE project_id = '${projectId}'+ AND id IN (SELECT DISTINCT session_id FROM events WHERE ${businessFilter})+ GROUP BY id) FORMAT TSV`+ ),+]);++const summary = {+ run: runId,+ completed,+ shape: {+ devices,+ perDevice,+ concurrency,+ properties: propertyCount,+ identified: values.identified,+ },+ total,+ accepted,+ failed,+ retried,+ seconds: Object.fromEntries(+ Object.entries(marks).map(([key, value]) => [key, Number(value.toFixed(1))])+ ),+ businessEventsPerSecond: Number((accepted / runSeconds).toFixed(0)),+ peaks,+ redisCpuSeconds: Number(redisCpu.toFixed(2)),+ redisForkCpuSeconds: Number(redisForkCpu.toFixed(2)),+ verify: {+ eventsAndUniqueIds: eventCounts,+ sessionStarts,+ sessionsAndUncollapsed: sessionCounts,+ expected: `${total} ${total} / ${devices} / ${devices} 0`,+ },+};+log(JSON.stringify(summary));+await Bun.write(`${outPrefix}.json`, JSON.stringify(summary, null, 2));+await admin.disconnect();+redis.disconnect();+if (!completed || failed > 0) {+ process.exit(1);+}diff --git a/apps/api/package.json b/apps/api/package.jsonindex 9cc8108d3..8063c1744 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -11,6 +11,7 @@"e2e:sessions": "LOG_LEVEL=warn dotenv -e ../../.env -- bun e2e/session-e2e.ts","e2e:sessions:stress": "LOG_LEVEL=warn dotenv -e ../../.env -- bun e2e/session-stress.ts","e2e:saturation": "LOG_LEVEL=warn dotenv -e ../../.env -- bun e2e/saturation-sweep.ts",+ "e2e:ingest-load": "LOG_LEVEL=warn dotenv -e ../../.env -- bun e2e/ingest-load.ts","e2e:legacy-jobs": "bash e2e/legacy-job-proof.sh","test": "bun test --isolate","typecheck": "tsc --noEmit"diff --git a/packages/core/src/buffers/base-buffer.test.ts b/packages/core/src/buffers/base-buffer.test.tsnew file mode 100644index 000000000..448876b03--- /dev/null+++ b/packages/core/src/buffers/base-buffer.test.ts@@ -0,0 +1,96 @@+import { beforeEach, describe, expect, it } from 'bun:test';+import { getRedisCache } from '@openpanel/redis';+import { stubBufferDeps } from '../../test/buffer-fixtures';+import { BaseBuffer, type FlushObservation } from './base-buffer';++const redis = getRedisCache();+const LOCK_TTL_SECONDS = 1;+const BATCH_SIZE = 2;++/** A buffer whose flush is whatever the test hands in, under a 1 s lock. */+class TestBuffer extends BaseBuffer {+ constructor(onFlush: () => Promise<void>) {+ super(stubBufferDeps, { name: 'lock-test', onFlush });+ this.lockTimeout = LOCK_TTL_SECONDS;+ }++ protected getRedisListKey(): string {+ return 'lock-test:queue';+ }++ drain(processBatch: () => Promise<number>): Promise<void> {+ return this.drainBatches(BATCH_SIZE, processBatch);+ }+}++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++beforeEach(async () => {+ await redis.del('lock:lock-test');+});++describe('BaseBuffer flush lock', () => {+ it('a flush that outlives the lock TTL keeps the lock until it is done', async () => {+ const ttlSamples: number[] = [];+ let concurrentResult: FlushObservation['result'] | undefined;+ const buffer = new TestBuffer(async () => {+ const otherWorker = new TestBuffer(() => Promise.resolve());+ otherWorker.flushObserver = (obs) => {+ concurrentResult = obs.result;+ };+ // Twice the TTL, sampling the lock on the way.+ for (let i = 0; i < 4; i++) {+ await sleep(LOCK_TTL_SECONDS * 500);+ ttlSamples.push(await redis.ttl('lock:lock-test'));+ }+ await otherWorker.tryFlush({ trigger: 'cron' });+ });++ await buffer.tryFlush({ trigger: 'cron' });++ expect(ttlSamples.every((ttl) => ttl > 0)).toBe(true);+ expect(concurrentResult).toBe('locked');+ expect(await redis.get('lock:lock-test')).toBeNull();+ });++ it('one instance runs one flush at a time, whatever the cron concurrency', async () => {+ let concurrentResult: FlushObservation['result'] | undefined;+ let flushes = 0;+ const buffer = new TestBuffer(async () => {+ flushes += 1;+ await sleep(200);+ });+ buffer.flushObserver = (obs) => {+ if (obs.result !== 'success') {+ concurrentResult = obs.result;+ }+ };++ await Promise.all([+ buffer.tryFlush({ trigger: 'cron' }),+ buffer.tryFlush({ trigger: 'cron' }),+ ]);++ expect(flushes).toBe(1);+ expect(concurrentResult).toBe('locked');+ });++ it('stops draining once the lock is no longer its own', async () => {+ let batches = 0;+ const processBatch = async () => {+ batches += 1;+ // Another holder took the key, as an expired lock would allow.+ await redis.set('lock:lock-test', 'someone-else');+ // Long enough for the keepalive to notice.+ await sleep(LOCK_TTL_SECONDS * 800);+ return BATCH_SIZE;+ };+ let buffer: TestBuffer;+ buffer = new TestBuffer(() => buffer.drain(processBatch));++ await buffer.tryFlush({ trigger: 'cron' });++ expect(batches).toBe(1);+ expect(await redis.get('lock:lock-test')).toBe('someone-else');+ });+});diff --git a/packages/core/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.tsindex 84bde8ade..018440046 100644--- a/packages/core/src/buffers/base-buffer.ts+++ b/packages/core/src/buffers/base-buffer.ts@@ -45,6 +45,13 @@ const DEFAULT_CH_INSERT_CONCURRENCY = 5;/** One cron run drains as many batches as fit before the next run is due. */const DRAIN_TIME_BUDGET_MS = 8000;+/**+ * The flush lock is refreshed this many times per TTL. The TTL only has to+ * outlive a crashed worker; a live flush keeps extending it, so a slow+ * ClickHouse can never let a second flush start on the same list.+ */+const LOCK_KEEPALIVE_PER_TTL = 3;+const releaseLockScript = defineRedisScript(`if redis.call("get", KEYS[1]) == ARGV[1] thenreturn redis.call("del", KEYS[1])@@ -53,7 +60,15 @@ elseend`);-export type FlushPhaseTimings = {+const extendLockScript = defineRedisScript(`+if redis.call("get", KEYS[1]) == ARGV[1] then+ return redis.call("expire", KEYS[1], ARGV[2])+else+ return 0+end+`);++export interface FlushPhaseTimings {lrangeMs?: number;/** Time spent in CH SELECT(s) during flush (e.g. profile-buffer's* fetch-existing-profile-for-merge step). Only buffers that read CH@@ -65,7 +80,7 @@ export type FlushPhaseTimings = {trimMs?: number;/** Total time spent inside onFlush (subclass-specific processing) */onFlushMs?: number;-};+}const FLUSH_PHASES = ['lrangeMs',@@ -78,7 +93,7 @@ const FLUSH_PHASES = [export type FlushTrigger = 'add' | 'cron';-export type FlushObservation = {+export interface FlushObservation {buffer: string;/*** - `success`: flush ran and completed without error@@ -97,11 +112,11 @@ export type FlushObservation = {llenAtStart?: number;phases?: FlushPhaseTimings;err?: unknown;-};+}export type FlushObserver = (obs: FlushObservation) => void;-export type AddObservation = {+export interface AddObservation {buffer: string;durationMs: number;/** True if the add was a no-op (e.g. cached-and-skipped). When set,@@ -110,7 +125,7 @@ export type AddObservation = {skipped?: boolean;/** Free-form reason label for skipped adds (e.g. 'cached'). */skipReason?: string;-};+}export type AddObserver = (obs: AddObservation) => void;@@ -118,7 +133,21 @@ export class BaseBuffer {name: string;logger: Logger;lockKey: string;+ /** Seconds. How long a crashed worker's lock blocks the next flush. */lockTimeout = 60;+ /**+ * Set by the keepalive when the lock is no longer this flush's: expired+ * (Redis unreachable for longer than the TTL) or taken by someone else.+ * `drainBatches` stops before its next batch, because two flushes on one+ * list insert the same batch twice and then trim a batch nobody inserted.+ */+ private lockLost = false;+ /**+ * One flush per buffer per process. The Redis lock keeps workers apart; this+ * keeps the cron's concurrent runs apart, so `lockLost` always describes the+ * one flush this instance is running.+ */+ private flushInFlight = false;onFlush: () => Promise<void> | void;enableParallelProcessing: boolean;@@ -263,7 +292,7 @@ export class BaseBuffer {}protected chunks<T>(items: T[], size: number) {- const chunks = [];+ const chunks: T[][] = [];for (let i = 0; i < items.length; i += size) {chunks.push(items.slice(i, i + size));}@@ -356,7 +385,7 @@ export class BaseBuffer {processBatch: () => Promise<number>): Promise<void> {const deadline = performance.now() + DRAIN_TIME_BUDGET_MS;- while (performance.now() < deadline) {+ while (performance.now() < deadline && !this.lockLost) {const processed = await processBatch();if (processed < batchSize) {return;@@ -407,6 +436,51 @@ export class BaseBuffer {await releaseLockScript(getRedisCache(), [this.lockKey], [lockId]);}+ /** Returns the function that ends the keepalive for this flush. */+ private keepLockAlive(lockId: string): () => void {+ const intervalMs = (this.lockTimeout * 1000) / LOCK_KEEPALIVE_PER_TTL;+ let ownershipLost = false;+ // A renewal still in flight when the flush ends must not touch the flag+ // of the next flush, which may already hold a new lock.+ let stopped = false;+ const interval = setInterval(async () => {+ if (stopped || ownershipLost) {+ return;+ }+ try {+ const extended = await extendLockScript(+ getRedisCache(),+ [this.lockKey],+ [lockId, String(this.lockTimeout)]+ );+ if (stopped || extended === 1) {+ return;+ }+ ownershipLost = true;+ this.lockLost = true;+ this.logger.error(+ { lockKey: this.lockKey },+ 'flush lock lost mid-flush; stopping before the next batch'+ );+ } catch (error) {+ if (stopped) {+ return;+ }+ // A failed call says nothing about ownership: the key may still be+ // ours, so keep renewing for the batch in flight and start no new one.+ this.lockLost = true;+ this.logger.error(+ { err: error, lockKey: this.lockKey },+ 'flush lock could not be extended; stopping before the next batch'+ );+ }+ }, intervalMs);+ return () => {+ stopped = true;+ clearInterval(interval);+ };+ }+/*** Emit a flush observation to the configured observer and to the structured* logger. Catches observer errors so a bad metrics path can never break@@ -526,6 +600,29 @@ export class BaseBuffer {return;}+ if (this.flushInFlight) {+ this.emitFlushObservation({+ buffer: this.name,+ result: 'locked',+ trigger,+ totalMs: performance.now() - startedAt,+ llenAtStart,+ });+ return;+ }+ this.flushInFlight = true;+ try {+ await this.flushUnderLock(trigger, startedAt, llenAtStart);+ } finally {+ this.flushInFlight = false;+ }+ }++ private async flushUnderLock(+ trigger: FlushTrigger,+ startedAt: number,+ llenAtStart: number | undefined+ ): Promise<void> {const lockId = generateSecureId('lock');const acquired = await getRedisCache().set(this.lockKey,@@ -546,6 +643,8 @@ export class BaseBuffer {return;}+ this.lockLost = false;+ const stopKeepAlive = this.keepLockAlive(lockId);const onFlushStarted = performance.now();try {await this.onFlush();@@ -576,6 +675,7 @@ export class BaseBuffer {err: error,});} finally {+ stopKeepAlive();await this.releaseLock(lockId);}}