Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M18-005): Fix M18-001's false-negative flush gate: resolve on a durability WATERMARK, not on whether this caller happened to start the write

ralph714dae9df650
diff --git a/packages/core/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.ts
index 0be2de597..feee6cb23 100644
--- a/packages/core/src/buffers/event-buffer.test.ts
+++ b/packages/core/src/buffers/event-buffer.test.ts
@@ -428,6 +428,117 @@ describe('EventBuffer', () => {
});
});
+ // M18-005 / drill 03 re-run: the gate has to answer "are my events in
+ // Redis?", not "did I start the write?". Answering the second question made
+ // 221 durability failures in 5.5 minutes on a HEALTHY stack and turned 16.9%
+ // of accepted events into duplicates, because a caller that lost the race to
+ // start a write was told its already-durable events were not durable.
+ describe('flushPendingOrThrow under concurrency', () => {
+ // 24 partitions is what drill 03 ran; the callers are its `eachBatch`
+ // handlers, and the background producer stands in for the HTTP path and
+ // the session-end job adding events underneath them.
+ const CONCURRENT_CALLERS = 24;
+ const ROUNDS_PER_CALLER = 25;
+ const EVENTS_PER_ROUND = 5;
+
+ let addedEventCount = 0;
+
+ const addEvent = (name: string) => {
+ addedEventCount += 1;
+ eventBuffer.add({
+ project_id: 'p17',
+ name,
+ created_at: new Date().toISOString(),
+ } as any);
+ };
+
+ beforeEach(() => {
+ addedEventCount = 0;
+ });
+
+ afterEach(async () => {
+ await redis.del('event_buffer:queue');
+ for (let attempt = 0; attempt < 5; attempt++) {
+ if (eventBuffer.getPendingLocalCount() === 0) {
+ break;
+ }
+ await eventBuffer.flush();
+ }
+ await redis.del('event_buffer:queue');
+ });
+
+ it('resolves every concurrent caller, with zero spurious rejections', async () => {
+ let producing = true;
+ const backgroundProducer = (async () => {
+ while (producing) {
+ addEvent('background');
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ }
+ })();
+
+ const rejections: unknown[] = [];
+ await Promise.all(
+ Array.from({ length: CONCURRENT_CALLERS }, async () => {
+ for (let round = 0; round < ROUNDS_PER_CALLER; round++) {
+ for (let index = 0; index < EVENTS_PER_ROUND; index++) {
+ addEvent('batch');
+ }
+ try {
+ await eventBuffer.flushPendingOrThrow();
+ } catch (error) {
+ rejections.push(error);
+ }
+ }
+ })
+ );
+ producing = false;
+ await backgroundProducer;
+
+ expect(rejections).toEqual([]);
+ await eventBuffer.flushPendingOrThrow();
+ expect(eventBuffer.getPendingLocalCount()).toBe(0);
+ expect(await eventBuffer.getBufferSize()).toBe(addedEventCount);
+ });
+
+ it('resolves a caller whose events someone else is already writing', async () => {
+ addEvent('written-by-someone-else');
+ addEvent('also-written-by-someone-else');
+
+ // `flush()` starts the write synchronously, and a write always takes the
+ // WHOLE pending array — so by the time this caller asks, its events are
+ // in a write it did not start. That is a success, not a failure.
+ const someoneElsesWrite = eventBuffer.flush();
+ expect(eventBuffer.getPendingLocalCount()).toBe(0);
+
+ await eventBuffer.flushPendingOrThrow();
+
+ expect(await eventBuffer.getBufferSize()).toBe(addedEventCount);
+ await someoneElsesWrite;
+ });
+
+ it('still rejects every concurrent caller when Redis genuinely refuses', async () => {
+ await redis.set('event_buffer:queue', 'not a list');
+
+ const results = await Promise.allSettled(
+ Array.from({ length: CONCURRENT_CALLERS }, async () => {
+ addEvent('doomed');
+ await eventBuffer.flushPendingOrThrow();
+ })
+ );
+
+ expect(results.every((result) => result.status === 'rejected')).toBe(
+ true
+ );
+ for (const result of results) {
+ expect((result as PromiseRejectedResult).reason.message).toContain(
+ 'WRONGTYPE'
+ );
+ }
+ // Nothing was dropped on the way to being refused.
+ expect(eventBuffer.getPendingLocalCount()).toBe(addedEventCount);
+ });
+ });
+
it('retains events in queue when ClickHouse insert fails', async () => {
eventBuffer.add({
project_id: 'p12',
diff --git a/packages/core/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.ts
index ccb888168..7636bf1b5 100644
--- a/packages/core/src/buffers/event-buffer.ts
+++ b/packages/core/src/buffers/event-buffer.ts
@@ -59,14 +59,6 @@ const DEFAULT_CHUNK_SIZE = 1000;
const DEFAULT_MICRO_BATCH_MS = 10;
const DEFAULT_MICRO_BATCH_SIZE = 100;
-/**
- * How many times `drainPendingToRedis` will wait for someone else's write
- * before it gives up on starting its own. Two passes is the normal worst case
- * (wait for the in-flight write, then start one); the third is margin against
- * a concurrent producer slipping a write in between.
- */
-const MAX_FLUSH_PASSES = 3;
-
export class EventBuffer extends BaseBuffer {
private readonly batchSize =
this.deps.config.buffers.event.batchSize ?? DEFAULT_BATCH_SIZE;
@@ -80,14 +72,29 @@ export class EventBuffer extends BaseBuffer {
private pendingEvents: IClickhouseEvent[] = [];
private flushTimer: ReturnType<typeof setTimeout> | null = null;
- private isFlushing = false;
+
/**
- * The micro-batch write currently in flight, so a caller that arrives while
- * `isFlushing` is set waits for its result instead of being told there was
- * nothing to do. The Kafka batch handler and shutdown are the callers that
- * cannot tolerate that answer.
+ * The durability watermark. Every event gets a monotonically increasing
+ * sequence number in `add()`; `lastDurableSeq` is the highest sequence a
+ * COMPLETED rpush covered. A caller's events are in Redis once
+ * `lastDurableSeq` reaches the sequence of its last event — whoever wrote
+ * them. That is the question the Kafka batch handler and shutdown need
+ * answered ("are my events in Redis?"), and it is not the same question as
+ * "did I start the write?" (M18-005, drill 03 re-run).
*/
- private inFlightFlush: Promise<unknown> | null = null;
+ private lastQueuedSeq = 0;
+ private lastDurableSeq = 0;
+
+ /**
+ * The rpush in flight and the highest sequence it will make durable. A write
+ * always takes the whole pending array, so a caller whose target sequence is
+ * at or below `coversSeq` is answered by this write and must not start
+ * another.
+ */
+ private inFlightWrite: {
+ coversSeq: number;
+ done: Promise<unknown>;
+ } | null = null;
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
@@ -112,6 +119,7 @@ export class EventBuffer extends BaseBuffer {
// Event-buffer's add() is synchronous (in-memory push). Measured anyway
// for consistency with the other buffers' add-latency tracking.
const start = performance.now();
+ this.lastQueuedSeq += 1;
this.pendingEvents.push(event);
if (this.pendingEvents.length >= this.microBatchMaxSize) {
@@ -139,90 +147,105 @@ export class EventBuffer extends BaseBuffer {
}
public async flush() {
- await this.drainPendingToRedis();
+ try {
+ await this.flushPendingOrThrow();
+ } catch {
+ // The fire-and-forget path swallows a Redis failure on purpose: the
+ // events are re-queued and the micro-batch timer tries again, which is
+ // the right answer while the process lives.
+ }
}
/**
* The same flush, but it THROWS when the rpush did not land, and it does not
- * resolve until every event that was pending when it was called is in Redis.
+ * resolve until every event accepted before the call is in Redis.
*
- * `flush()` swallows a Redis failure on purpose — it re-queues the events and
- * the micro-batch timer tries again, which is right while the process lives.
- * The two callers here have no next attempt: the Kafka batch handler is about
- * to resolve offsets that kafkajs commits as soon as it returns, and shutdown
- * is about to exit. Both must be able to SEE the failure and decline to
- * commit — a redelivered duplicate is recoverable, a dropped `pendingEvents`
- * array is not (M18-001, drill 03).
+ * `flush()` swallows a Redis failure on purpose. The two callers here have no
+ * next attempt: the Kafka batch handler is about to resolve offsets that
+ * kafkajs commits as soon as it returns, and shutdown is about to exit. Both
+ * must be able to SEE the failure and decline to commit — a redelivered
+ * duplicate is recoverable, a dropped `pendingEvents` array is not
+ * (M18-001, drill 03).
*/
public async flushPendingOrThrow(): Promise<void> {
- const failure = await this.drainPendingToRedis();
- if (failure === null) {
- return;
- }
- throw failure instanceof Error ? failure : new Error(String(failure));
- }
-
- /**
- * A write always takes the WHOLE pending array, so one write STARTED after
- * this call covers every event that was pending when it was made. A write
- * already in flight does not — it may have begun before the last `add()` —
- * so wait that one out and then start our own. Deliberately not "loop until
- * the buffer is empty": another producer (a session-end job) adding events
- * concurrently must not be able to turn a successful flush into a failure.
- *
- * @returns the failure the rpush ended with, or `null` on success.
- */
- private async drainPendingToRedis(): Promise<unknown> {
- for (let pass = 0; pass < MAX_FLUSH_PASSES; pass++) {
- if (this.flushTimer) {
- clearTimeout(this.flushTimer);
- this.flushTimer = null;
+ const target = this.lastQueuedSeq;
+
+ // Not a retry loop: each pass either waits out a write that is already
+ // running or starts the one write still missing, and a write covers
+ // everything pending when it starts. So a concurrent write that happens to
+ // carry our events resolves us — losing the race to start a write is a
+ // SUCCESS, which is the whole correctness point (M18-005).
+ while (this.lastDurableSeq < target) {
+ const inFlight = this.inFlightWrite;
+
+ if (inFlight === null) {
+ if (this.pendingEvents.length === 0) {
+ // Nothing pending and nothing in flight, so every sequence ever
+ // queued has been written. Unreachable unless the watermark and the
+ // array disagree; returning beats spinning.
+ return;
+ }
+ this.throwIfFailed(await this.flushLocalBuffer());
+ continue;
}
- const nothingToDrain =
- !this.isFlushing && this.pendingEvents.length === 0;
- if (nothingToDrain) {
- return null;
- }
-
- const startsItsOwnWrite = !this.isFlushing;
- const failure = await this.flushLocalBuffer();
- if (failure !== null) {
- return failure;
- }
- if (startsItsOwnWrite) {
- return null;
+ const failure = await inFlight.done;
+ // A write that did not carry our events is not our failure: its events
+ // are re-queued at the front and the write we start next takes them too.
+ if (inFlight.coversSeq >= target) {
+ this.throwIfFailed(failure);
}
}
+ }
- return new Error(
- `event buffer could not start its own flush within ${MAX_FLUSH_PASSES} passes`
- );
+ private throwIfFailed(failure: unknown): void {
+ if (failure === null) {
+ return;
+ }
+ throw failure instanceof Error ? failure : new Error(String(failure));
}
/**
+ * Starts a write unless one is already running, and returns the write that
+ * makes everything currently pending durable.
+ *
* Never rejects — `add()` calls it without awaiting. The failure comes back
- * as the resolved value instead, for the one caller that must act on it.
+ * as the resolved value instead, for the callers that must act on it.
*/
private flushLocalBuffer(): Promise<unknown> {
- if (this.isFlushing) {
- return this.inFlightFlush ?? Promise.resolve(null);
+ if (this.inFlightWrite) {
+ return this.inFlightWrite.done;
}
if (this.pendingEvents.length === 0) {
return Promise.resolve(null);
}
- this.inFlightFlush = this.writePendingToRedis();
- return this.inFlightFlush;
- }
-
- private async writePendingToRedis(): Promise<unknown> {
- this.isFlushing = true;
+ // The write takes everything the timer would have taken.
+ if (this.flushTimer) {
+ clearTimeout(this.flushTimer);
+ this.flushTimer = null;
+ }
const eventsToFlush = this.pendingEvents;
this.pendingEvents = [];
- let failure: unknown = null;
+ // The whole pending array goes in one write, and `add()` assigns sequences
+ // in order, so this write covers every sequence assigned so far.
+ const coversSeq = this.lastQueuedSeq;
+
+ const done = this.writeToRedis(eventsToFlush, coversSeq).then((failure) => {
+ this.inFlightWrite = null;
+ this.scheduleFlushIfPending();
+ return failure;
+ });
+ this.inFlightWrite = { coversSeq, done };
+ return done;
+ }
+ /** @returns the failure the rpush ended with, or `null` on success. */
+ private async writeToRedis(
+ eventsToFlush: IClickhouseEvent[],
+ coversSeq: number
+ ): Promise<unknown> {
try {
const redis = getRedisCache();
const multi = redis.multi();
@@ -245,9 +268,12 @@ export class EventBuffer extends BaseBuffer {
throw rejected[0];
}
+ // The durability boundary: past this point the events survive any
+ // process death, so everyone waiting on a sequence in this write is done.
+ this.lastDurableSeq = Math.max(this.lastDurableSeq, coversSeq);
this.flushRetryCount = 0;
+ return null;
} catch (error) {
- failure = error;
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -260,19 +286,19 @@ export class EventBuffer extends BaseBuffer {
},
'Failed to flush local buffer to Redis; events re-queued'
);
- } finally {
- this.isFlushing = false;
- this.inFlightFlush = null;
- // Events may have accumulated while we were flushing; schedule another flush if needed
- if (this.pendingEvents.length > 0 && !this.flushTimer) {
- this.flushTimer = setTimeout(() => {
- this.flushTimer = null;
- this.flushLocalBuffer();
- }, this.microBatchIntervalMs);
- }
+ return error;
}
+ }
- return failure;
+ /** Events may arrive while a write is running; they get the next micro-batch. */
+ private scheduleFlushIfPending(): void {
+ if (this.pendingEvents.length === 0 || this.flushTimer) {
+ return;
+ }
+ this.flushTimer = setTimeout(() => {
+ this.flushTimer = null;
+ this.flushLocalBuffer();
+ }, this.microBatchIntervalMs);
}
protected getRedisListKey(): string {