OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M18-005): Fix M18-001's false-negative flush gate: resolve on a durability WATERMARK, not on whether this caller happened to start the write
ralph714dae9df650
diff --git a/packages/core/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.tsindex 0be2de597..feee6cb23 100644--- a/packages/core/src/buffers/event-buffer.test.ts+++ b/packages/core/src/buffers/event-buffer.test.ts@@ -428,6 +428,117 @@ describe('EventBuffer', () => {});});+ // M18-005 / drill 03 re-run: the gate has to answer "are my events in+ // Redis?", not "did I start the write?". Answering the second question made+ // 221 durability failures in 5.5 minutes on a HEALTHY stack and turned 16.9%+ // of accepted events into duplicates, because a caller that lost the race to+ // start a write was told its already-durable events were not durable.+ describe('flushPendingOrThrow under concurrency', () => {+ // 24 partitions is what drill 03 ran; the callers are its `eachBatch`+ // handlers, and the background producer stands in for the HTTP path and+ // the session-end job adding events underneath them.+ const CONCURRENT_CALLERS = 24;+ const ROUNDS_PER_CALLER = 25;+ const EVENTS_PER_ROUND = 5;++ let addedEventCount = 0;++ const addEvent = (name: string) => {+ addedEventCount += 1;+ eventBuffer.add({+ project_id: 'p17',+ name,+ created_at: new Date().toISOString(),+ } as any);+ };++ beforeEach(() => {+ addedEventCount = 0;+ });++ afterEach(async () => {+ await redis.del('event_buffer:queue');+ for (let attempt = 0; attempt < 5; attempt++) {+ if (eventBuffer.getPendingLocalCount() === 0) {+ break;+ }+ await eventBuffer.flush();+ }+ await redis.del('event_buffer:queue');+ });++ it('resolves every concurrent caller, with zero spurious rejections', async () => {+ let producing = true;+ const backgroundProducer = (async () => {+ while (producing) {+ addEvent('background');+ await new Promise((resolve) => setTimeout(resolve, 0));+ }+ })();++ const rejections: unknown[] = [];+ await Promise.all(+ Array.from({ length: CONCURRENT_CALLERS }, async () => {+ for (let round = 0; round < ROUNDS_PER_CALLER; round++) {+ for (let index = 0; index < EVENTS_PER_ROUND; index++) {+ addEvent('batch');+ }+ try {+ await eventBuffer.flushPendingOrThrow();+ } catch (error) {+ rejections.push(error);+ }+ }+ })+ );+ producing = false;+ await backgroundProducer;++ expect(rejections).toEqual([]);+ await eventBuffer.flushPendingOrThrow();+ expect(eventBuffer.getPendingLocalCount()).toBe(0);+ expect(await eventBuffer.getBufferSize()).toBe(addedEventCount);+ });++ it('resolves a caller whose events someone else is already writing', async () => {+ addEvent('written-by-someone-else');+ addEvent('also-written-by-someone-else');++ // `flush()` starts the write synchronously, and a write always takes the+ // WHOLE pending array — so by the time this caller asks, its events are+ // in a write it did not start. That is a success, not a failure.+ const someoneElsesWrite = eventBuffer.flush();+ expect(eventBuffer.getPendingLocalCount()).toBe(0);++ await eventBuffer.flushPendingOrThrow();++ expect(await eventBuffer.getBufferSize()).toBe(addedEventCount);+ await someoneElsesWrite;+ });++ it('still rejects every concurrent caller when Redis genuinely refuses', async () => {+ await redis.set('event_buffer:queue', 'not a list');++ const results = await Promise.allSettled(+ Array.from({ length: CONCURRENT_CALLERS }, async () => {+ addEvent('doomed');+ await eventBuffer.flushPendingOrThrow();+ })+ );++ expect(results.every((result) => result.status === 'rejected')).toBe(+ true+ );+ for (const result of results) {+ expect((result as PromiseRejectedResult).reason.message).toContain(+ 'WRONGTYPE'+ );+ }+ // Nothing was dropped on the way to being refused.+ expect(eventBuffer.getPendingLocalCount()).toBe(addedEventCount);+ });+ });+it('retains events in queue when ClickHouse insert fails', async () => {eventBuffer.add({project_id: 'p12',diff --git a/packages/core/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.tsindex ccb888168..7636bf1b5 100644--- a/packages/core/src/buffers/event-buffer.ts+++ b/packages/core/src/buffers/event-buffer.ts@@ -59,14 +59,6 @@ const DEFAULT_CHUNK_SIZE = 1000;const DEFAULT_MICRO_BATCH_MS = 10;const DEFAULT_MICRO_BATCH_SIZE = 100;-/**- * How many times `drainPendingToRedis` will wait for someone else's write- * before it gives up on starting its own. Two passes is the normal worst case- * (wait for the in-flight write, then start one); the third is margin against- * a concurrent producer slipping a write in between.- */-const MAX_FLUSH_PASSES = 3;-export class EventBuffer extends BaseBuffer {private readonly batchSize =this.deps.config.buffers.event.batchSize ?? DEFAULT_BATCH_SIZE;@@ -80,14 +72,29 @@ export class EventBuffer extends BaseBuffer {private pendingEvents: IClickhouseEvent[] = [];private flushTimer: ReturnType<typeof setTimeout> | null = null;- private isFlushing = false;+/**- * The micro-batch write currently in flight, so a caller that arrives while- * `isFlushing` is set waits for its result instead of being told there was- * nothing to do. The Kafka batch handler and shutdown are the callers that- * cannot tolerate that answer.+ * The durability watermark. Every event gets a monotonically increasing+ * sequence number in `add()`; `lastDurableSeq` is the highest sequence a+ * COMPLETED rpush covered. A caller's events are in Redis once+ * `lastDurableSeq` reaches the sequence of its last event — whoever wrote+ * them. That is the question the Kafka batch handler and shutdown need+ * answered ("are my events in Redis?"), and it is not the same question as+ * "did I start the write?" (M18-005, drill 03 re-run).*/- private inFlightFlush: Promise<unknown> | null = null;+ private lastQueuedSeq = 0;+ private lastDurableSeq = 0;++ /**+ * The rpush in flight and the highest sequence it will make durable. A write+ * always takes the whole pending array, so a caller whose target sequence is+ * at or below `coversSeq` is answered by this write and must not start+ * another.+ */+ private inFlightWrite: {+ coversSeq: number;+ done: Promise<unknown>;+ } | null = null;/** Tracks consecutive flush failures for observability; reset on success. */private flushRetryCount = 0;@@ -112,6 +119,7 @@ export class EventBuffer extends BaseBuffer {// Event-buffer's add() is synchronous (in-memory push). Measured anyway// for consistency with the other buffers' add-latency tracking.const start = performance.now();+ this.lastQueuedSeq += 1;this.pendingEvents.push(event);if (this.pendingEvents.length >= this.microBatchMaxSize) {@@ -139,90 +147,105 @@ export class EventBuffer extends BaseBuffer {}public async flush() {- await this.drainPendingToRedis();+ try {+ await this.flushPendingOrThrow();+ } catch {+ // The fire-and-forget path swallows a Redis failure on purpose: the+ // events are re-queued and the micro-batch timer tries again, which is+ // the right answer while the process lives.+ }}/*** The same flush, but it THROWS when the rpush did not land, and it does not- * resolve until every event that was pending when it was called is in Redis.+ * resolve until every event accepted before the call is in Redis.*- * `flush()` swallows a Redis failure on purpose — it re-queues the events and- * the micro-batch timer tries again, which is right while the process lives.- * The two callers here have no next attempt: the Kafka batch handler is about- * to resolve offsets that kafkajs commits as soon as it returns, and shutdown- * is about to exit. Both must be able to SEE the failure and decline to- * commit — a redelivered duplicate is recoverable, a dropped `pendingEvents`- * array is not (M18-001, drill 03).+ * `flush()` swallows a Redis failure on purpose. The two callers here have no+ * next attempt: the Kafka batch handler is about to resolve offsets that+ * kafkajs commits as soon as it returns, and shutdown is about to exit. Both+ * must be able to SEE the failure and decline to commit — a redelivered+ * duplicate is recoverable, a dropped `pendingEvents` array is not+ * (M18-001, drill 03).*/public async flushPendingOrThrow(): Promise<void> {- const failure = await this.drainPendingToRedis();- if (failure === null) {- return;- }- throw failure instanceof Error ? failure : new Error(String(failure));- }-- /**- * A write always takes the WHOLE pending array, so one write STARTED after- * this call covers every event that was pending when it was made. A write- * already in flight does not — it may have begun before the last `add()` —- * so wait that one out and then start our own. Deliberately not "loop until- * the buffer is empty": another producer (a session-end job) adding events- * concurrently must not be able to turn a successful flush into a failure.- *- * @returns the failure the rpush ended with, or `null` on success.- */- private async drainPendingToRedis(): Promise<unknown> {- for (let pass = 0; pass < MAX_FLUSH_PASSES; pass++) {- if (this.flushTimer) {- clearTimeout(this.flushTimer);- this.flushTimer = null;+ const target = this.lastQueuedSeq;++ // Not a retry loop: each pass either waits out a write that is already+ // running or starts the one write still missing, and a write covers+ // everything pending when it starts. So a concurrent write that happens to+ // carry our events resolves us — losing the race to start a write is a+ // SUCCESS, which is the whole correctness point (M18-005).+ while (this.lastDurableSeq < target) {+ const inFlight = this.inFlightWrite;++ if (inFlight === null) {+ if (this.pendingEvents.length === 0) {+ // Nothing pending and nothing in flight, so every sequence ever+ // queued has been written. Unreachable unless the watermark and the+ // array disagree; returning beats spinning.+ return;+ }+ this.throwIfFailed(await this.flushLocalBuffer());+ continue;}- const nothingToDrain =- !this.isFlushing && this.pendingEvents.length === 0;- if (nothingToDrain) {- return null;- }-- const startsItsOwnWrite = !this.isFlushing;- const failure = await this.flushLocalBuffer();- if (failure !== null) {- return failure;- }- if (startsItsOwnWrite) {- return null;+ const failure = await inFlight.done;+ // A write that did not carry our events is not our failure: its events+ // are re-queued at the front and the write we start next takes them too.+ if (inFlight.coversSeq >= target) {+ this.throwIfFailed(failure);}}+ }- return new Error(- `event buffer could not start its own flush within ${MAX_FLUSH_PASSES} passes`- );+ private throwIfFailed(failure: unknown): void {+ if (failure === null) {+ return;+ }+ throw failure instanceof Error ? failure : new Error(String(failure));}/**+ * Starts a write unless one is already running, and returns the write that+ * makes everything currently pending durable.+ ** Never rejects — `add()` calls it without awaiting. The failure comes back- * as the resolved value instead, for the one caller that must act on it.+ * as the resolved value instead, for the callers that must act on it.*/private flushLocalBuffer(): Promise<unknown> {- if (this.isFlushing) {- return this.inFlightFlush ?? Promise.resolve(null);+ if (this.inFlightWrite) {+ return this.inFlightWrite.done;}if (this.pendingEvents.length === 0) {return Promise.resolve(null);}- this.inFlightFlush = this.writePendingToRedis();- return this.inFlightFlush;- }-- private async writePendingToRedis(): Promise<unknown> {- this.isFlushing = true;+ // The write takes everything the timer would have taken.+ if (this.flushTimer) {+ clearTimeout(this.flushTimer);+ this.flushTimer = null;+ }const eventsToFlush = this.pendingEvents;this.pendingEvents = [];- let failure: unknown = null;+ // The whole pending array goes in one write, and `add()` assigns sequences+ // in order, so this write covers every sequence assigned so far.+ const coversSeq = this.lastQueuedSeq;++ const done = this.writeToRedis(eventsToFlush, coversSeq).then((failure) => {+ this.inFlightWrite = null;+ this.scheduleFlushIfPending();+ return failure;+ });+ this.inFlightWrite = { coversSeq, done };+ return done;+ }+ /** @returns the failure the rpush ended with, or `null` on success. */+ private async writeToRedis(+ eventsToFlush: IClickhouseEvent[],+ coversSeq: number+ ): Promise<unknown> {try {const redis = getRedisCache();const multi = redis.multi();@@ -245,9 +268,12 @@ export class EventBuffer extends BaseBuffer {throw rejected[0];}+ // The durability boundary: past this point the events survive any+ // process death, so everyone waiting on a sequence in this write is done.+ this.lastDurableSeq = Math.max(this.lastDurableSeq, coversSeq);this.flushRetryCount = 0;+ return null;} catch (error) {- failure = error;// Re-queue failed events at the front to preserve order and avoid data lossthis.pendingEvents = eventsToFlush.concat(this.pendingEvents);@@ -260,19 +286,19 @@ export class EventBuffer extends BaseBuffer {},'Failed to flush local buffer to Redis; events re-queued');- } finally {- this.isFlushing = false;- this.inFlightFlush = null;- // Events may have accumulated while we were flushing; schedule another flush if needed- if (this.pendingEvents.length > 0 && !this.flushTimer) {- this.flushTimer = setTimeout(() => {- this.flushTimer = null;- this.flushLocalBuffer();- }, this.microBatchIntervalMs);- }+ return error;}+ }- return failure;+ /** Events may arrive while a write is running; they get the next micro-batch. */+ private scheduleFlushIfPending(): void {+ if (this.pendingEvents.length === 0 || this.flushTimer) {+ return;+ }+ this.flushTimer = setTimeout(() => {+ this.flushTimer = null;+ this.flushLocalBuffer();+ }, this.microBatchIntervalMs);}protected getRedisListKey(): string {