Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M18-007): Stop the redelivery amplification: drop the re-queue that M18-001 made double-counting, and add backoff so a failed flush cannot spin

ralph22f925693703
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index a0f27d8be..efb5f9fa0 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -271,10 +271,11 @@ async function startIngestConsumer(
}),
sendToDeadLetter: (message) =>
produceDeadLetterEvent(config.core, message),
- // Runs before the batch's offsets are resolved, and THROWS if the rpush
- // did not land. `pendingEvents` is the only in-process buffer state in
- // the tree — every other buffer writes to Redis inside `add()`.
- flushBufferedEvents: () => deps.buffers.event.flushPendingOrThrow(),
+ // Opened before the batch's first handler and closed before its first
+ // resolved offset; the gate THROWS if the rpush did not land.
+ // `pendingEvents` is the only in-process buffer state in the tree —
+ // every other buffer writes to Redis inside `add()`.
+ openDurabilityWindow: () => deps.buffers.event.openDurabilityWindow(),
logger,
metrics: ingestConsumerMetrics,
onActivity: markEventsActivity,
diff --git a/packages/core/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.ts
index feee6cb23..2abb7ec7b 100644
--- a/packages/core/src/buffers/event-buffer.test.ts
+++ b/packages/core/src/buffers/event-buffer.test.ts
@@ -539,6 +539,110 @@ describe('EventBuffer', () => {
});
});
+ // M18-007 / drill 02 re-run: a failed flush leaves the batch's Kafka offsets
+ // unresolved (M18-001) and the fail-fast cache client (M18-003) brings the
+ // redelivery back in ~90ms. Re-queueing the failed events then holds one
+ // copy per lap — 4,759 ClickHouse rows for 230 events, measured.
+ describe('events a redelivery will produce again', () => {
+ const REDELIVERY_LAPS = 5;
+
+ /** One Kafka message's event. A redelivery builds an equal, NEW object. */
+ const kafkaEvent = (name = 'redelivered') =>
+ ({
+ project_id: 'p18',
+ name,
+ created_at: '2026-09-13 20:50:31',
+ }) as any;
+
+ const queuedNames = async (): Promise<string[]> =>
+ (await redis.lrange('event_buffer:queue', 0, -1)).map(
+ (row) => (JSON.parse(row) as { name: string }).name
+ );
+
+ it('keeps ONE copy when the same message is redelivered through a whole outage', async () => {
+ await redis.set('event_buffer:queue', 'not a list');
+
+ for (let lap = 0; lap < REDELIVERY_LAPS; lap++) {
+ const isDurable = eventBuffer.openDurabilityWindow();
+ eventBuffer.addRedeliverable(kafkaEvent());
+ await expect(isDurable()).rejects.toThrow('WRONGTYPE');
+ // Nothing is held for a later retry: Kafka owns that now.
+ expect(eventBuffer.getPendingLocalCount()).toBe(0);
+ }
+
+ await redis.del('event_buffer:queue');
+ const isDurable = eventBuffer.openDurabilityWindow();
+ eventBuffer.addRedeliverable(kafkaEvent());
+ await isDurable();
+
+ expect(await eventBuffer.getBufferSize()).toBe(1);
+ });
+
+ it('still holds the events of a producer nothing redelivers', async () => {
+ await redis.set('event_buffer:queue', 'not a list');
+
+ const isDurable = eventBuffer.openDurabilityWindow();
+ eventBuffer.addRedeliverable(kafkaEvent('from-the-consumer'));
+ // The session-end job's write: a BullMQ retry re-runs a job whose Redis
+ // `SET NX` claim is already taken, so this copy is the only one there is.
+ eventBuffer.add(kafkaEvent('from-the-session-end-job'));
+
+ await expect(isDurable()).rejects.toThrow('WRONGTYPE');
+ expect(eventBuffer.getPendingLocalCount()).toBe(1);
+
+ await redis.del('event_buffer:queue');
+ await eventBuffer.flush();
+
+ expect(await queuedNames()).toEqual(['from-the-session-end-job']);
+ });
+
+ it('rejects the window whose events a write it never awaited dropped', async () => {
+ // The load-bearing case. The micro-batch timer can start — and fail —
+ // the write while the batch is still handling messages, so by the time
+ // the gate is asked there is nothing pending and Redis may be healthy
+ // again. Answering "everything queued is settled" would commit the
+ // batch and lose it for good.
+ await redis.set('event_buffer:queue', 'not a list');
+
+ const isDurable = eventBuffer.openDurabilityWindow();
+ eventBuffer.addRedeliverable(kafkaEvent());
+ await eventBuffer.flush();
+ expect(eventBuffer.getPendingLocalCount()).toBe(0);
+
+ await redis.del('event_buffer:queue');
+
+ await expect(isDurable()).rejects.toThrow('WRONGTYPE');
+ expect(await eventBuffer.getBufferSize()).toBe(0);
+ });
+
+ it('does not poison a window opened after the drop', async () => {
+ await redis.set('event_buffer:queue', 'not a list');
+ const doomed = eventBuffer.openDurabilityWindow();
+ eventBuffer.addRedeliverable(kafkaEvent('doomed'));
+ await expect(doomed()).rejects.toThrow('WRONGTYPE');
+
+ await redis.del('event_buffer:queue');
+ const healthy = eventBuffer.openDurabilityWindow();
+ eventBuffer.addRedeliverable(kafkaEvent('healthy'));
+ await healthy();
+
+ expect(await queuedNames()).toEqual(['healthy']);
+ });
+
+ it("routes the consumer scope's add() through addRedeliverable", async () => {
+ await redis.set('event_buffer:queue', 'not a list');
+ const consumerView = eventBuffer.asRedeliverable();
+
+ const isDurable = eventBuffer.openDurabilityWindow();
+ consumerView.add(kafkaEvent());
+ await expect(isDurable()).rejects.toThrow('WRONGTYPE');
+
+ // Same buffer underneath, and the event was dropped rather than held.
+ expect(consumerView.getPendingLocalCount()).toBe(0);
+ expect(eventBuffer.getPendingLocalCount()).toBe(0);
+ });
+ });
+
it('retains events in queue when ClickHouse insert fails', async () => {
eventBuffer.add({
project_id: 'p12',
diff --git a/packages/core/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.ts
index 7636bf1b5..6bc29589f 100644
--- a/packages/core/src/buffers/event-buffer.ts
+++ b/packages/core/src/buffers/event-buffer.ts
@@ -98,6 +98,28 @@ export class EventBuffer extends BaseBuffer {
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
+ /**
+ * Events whose producer re-produces them when this buffer reports a failed
+ * flush. They are DROPPED by a failed write instead of being re-queued —
+ * see `addRedeliverable`.
+ */
+ private readonly redeliverableEvents = new WeakSet<IClickhouseEvent>();
+
+ /**
+ * The highest sequence a failed write dropped, and the failure it dropped
+ * it with. `lastDurableSeq` answers "is everything up to here in Redis?",
+ * which stopped being the same question once a write could drop events: a
+ * later write can carry the watermark past a sequence that was dropped
+ * rather than written. A durability window (`openDurabilityWindow`) compares
+ * against this so the producer that owns the dropped events is the one told
+ * about it.
+ */
+ private droppedThroughSeq = 0;
+ private lastDropFailure: unknown = null;
+
+ /** Built once, on first use; see `asRedeliverable`. */
+ private redeliverableView: EventBuffer | null = null;
+
private queueKey = 'event_buffer:queue';
constructor(deps: BufferDeps) {
@@ -167,7 +189,9 @@ export class EventBuffer extends BaseBuffer {
* duplicate is recoverable, a dropped `pendingEvents` array is not
* (M18-001, drill 03).
*/
- public async flushPendingOrThrow(): Promise<void> {
+ public async flushPendingOrThrow(
+ sinceSeq: number = this.droppedThroughSeq
+ ): Promise<void> {
const target = this.lastQueuedSeq;
// Not a retry loop: each pass either waits out a write that is already
@@ -180,10 +204,11 @@ export class EventBuffer extends BaseBuffer {
if (inFlight === null) {
if (this.pendingEvents.length === 0) {
- // Nothing pending and nothing in flight, so every sequence ever
- // queued has been written. Unreachable unless the watermark and the
- // array disagree; returning beats spinning.
- return;
+ // Nothing pending and nothing in flight: every sequence queued is
+ // SETTLED — written, or dropped for the producer that owns it to
+ // produce again. Settled is not durable, which the check below is
+ // what separates.
+ break;
}
this.throwIfFailed(await this.flushLocalBuffer());
continue;
@@ -196,6 +221,72 @@ export class EventBuffer extends BaseBuffer {
this.throwIfFailed(failure);
}
}
+
+ // A write that dropped events did not make them durable, and the watermark
+ // alone cannot say so: it moves on with the NEXT write, which carries none
+ // of them. Without this a batch whose events were dropped by a micro-batch
+ // timer it never awaited would be told they are safe, commit its offsets,
+ // and lose them for good (M18-007).
+ if (this.droppedThroughSeq > sinceSeq) {
+ this.throwIfFailed(this.lastDropFailure);
+ }
+ }
+
+ /**
+ * Buffer an event whose producer produces it AGAIN when this buffer reports
+ * a failed flush — today the Kafka consumer, whose batch is left
+ * uncommitted and redelivered (`modules/ingest/src/consumer.ts`).
+ *
+ * Such an event is not re-queued by a failed write. Re-queueing it was
+ * right while the buffer owned the retry, and became harmful the moment
+ * Kafka took that ownership over (M18-001): the redelivery buffers a fresh
+ * copy, so the old one is a SECOND copy of one event, and every further
+ * redelivery adds another. Drill 02's re-run measured 4,759 ClickHouse rows
+ * for 230 events that way (M18-007).
+ *
+ * Producers Kafka does not redeliver — the session-end job, whose Redis
+ * `SET NX` claim makes a job retry a no-op — keep the safety net by using
+ * plain `add()`.
+ */
+ public addRedeliverable(event: IClickhouseEvent): void {
+ this.redeliverableEvents.add(event);
+ this.add(event);
+ }
+
+ /**
+ * A view of this buffer whose `add()` is `addRedeliverable()`; everything
+ * else on it is this buffer's own.
+ *
+ * The ownership has to ride in on the SCOPE because `createEvent()` reaches
+ * the buffer as `deps.buffers.event` and takes no ownership argument — so
+ * the consumer hands its handlers a scope carrying this view
+ * (`modules/ingest/src/consumer-handler.ts`) and every other transport keeps
+ * the buffer itself.
+ */
+ public asRedeliverable(): EventBuffer {
+ this.redeliverableView ??= new Proxy(this, {
+ get: (target, property) =>
+ property === 'add'
+ ? (event: IClickhouseEvent) => target.addRedeliverable(event)
+ : Reflect.get(target, property, target),
+ });
+ return this.redeliverableView;
+ }
+
+ /**
+ * Opens a durability window and returns the gate that closes it: a promise
+ * that resolves once everything buffered after this call is in Redis, and
+ * rejects when any of it was dropped instead.
+ *
+ * The LOWER bound is the point. A caller that asks "is everything queued in
+ * Redis?" cannot tell its own dropped events from another producer's, and a
+ * failed write can drop events the caller never awaited. Opening the window
+ * before the first event is buffered is what makes the answer that caller's
+ * own (M18-007).
+ */
+ public openDurabilityWindow(): () => Promise<void> {
+ const sinceSeq = this.lastQueuedSeq;
+ return () => this.flushPendingOrThrow(sinceSeq);
}
private throwIfFailed(failure: unknown): void {
@@ -274,14 +365,30 @@ export class EventBuffer extends BaseBuffer {
this.flushRetryCount = 0;
return null;
} catch (error) {
- // Re-queue failed events at the front to preserve order and avoid data loss
- this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
+ // Re-queue only the events nothing else will produce again, at the front
+ // to preserve order. A redeliverable event is already on its way back —
+ // keeping it here would hold a second copy of it until Redis returns,
+ // and one more for every redelivery in between (see `addRedeliverable`).
+ const orphaned = eventsToFlush.filter(
+ (event) => !this.redeliverableEvents.has(event)
+ );
+ const droppedCount = eventsToFlush.length - orphaned.length;
+ if (droppedCount > 0) {
+ // `coversSeq` over-states which sequences were dropped when the write
+ // mixed the two kinds. Erring high costs an extra redelivery for a
+ // window that straddled the failure; erring low would cost the events.
+ this.droppedThroughSeq = Math.max(this.droppedThroughSeq, coversSeq);
+ this.lastDropFailure = error;
+ }
+ this.pendingEvents = orphaned.concat(this.pendingEvents);
this.flushRetryCount += 1;
this.logger.warn(
{
err: error,
eventCount: eventsToFlush.length,
+ requeuedCount: orphaned.length,
+ droppedForRedeliveryCount: droppedCount,
flushRetryCount: this.flushRetryCount,
},
'Failed to flush local buffer to Redis; events re-queued'
diff --git a/packages/core/src/modules/ingest/src/consumer-handler.ts b/packages/core/src/modules/ingest/src/consumer-handler.ts
index 37dd152f3..29c118d75 100644
--- a/packages/core/src/modules/ingest/src/consumer-handler.ts
+++ b/packages/core/src/modules/ingest/src/consumer-handler.ts
@@ -29,9 +29,20 @@ export function createIncomingEventHandler(
deps: AppDeps,
bindings: IncomingEventBindings
) {
+ // Every event this handler buffers is produced again by a redelivery when
+ // the batch's durability gate fails, so the event buffer must not keep a
+ // re-queued copy of it too — `EventBuffer.addRedeliverable`. `createEvent()`
+ // reaches the buffer as `deps.buffers.event` and takes no ownership
+ // argument, so the ownership rides in on the scope. Built once, here, not
+ // per message: the hot path gets no work it did not already have.
+ const consumerDeps: AppDeps = {
+ ...deps,
+ buffers: { ...deps.buffers, event: deps.buffers.event.asRedeliverable() },
+ };
+
return (payload: IncomingEventPayload, meta: IncomingEventDelivery) => {
const requestId = incomingEventRequestId(payload);
- const ctx = createCtx(deps, {
+ const ctx = createCtx(consumerDeps, {
requestId,
logger: deps.logger.child({ [REQUEST_ID_LOG_FIELD]: requestId }),
});
diff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.ts
index b2f9a42f6..8f1d3e26e 100644
--- a/packages/core/src/modules/ingest/src/consumer.test.ts
+++ b/packages/core/src/modules/ingest/src/consumer.test.ts
@@ -63,7 +63,14 @@ interface MockCalls {
const resolvedOffsets = (resolveOffset: MockCalls): string[] =>
resolveOffset.mock.calls.map(([offset]) => offset as string);
-function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {
+type DepOverrides = Partial<
+ Omit<EventsBatchHandlerDeps, 'openDurabilityWindow'>
+> & {
+ /** Sugar: the gate every window in this test closes with. */
+ flushBufferedEvents?: () => Promise<void>;
+};
+
+function makeDeps(overrides: DepOverrides = {}) {
const metrics: EventsBatchHandlerDeps['metrics'] = {
reprocessed: mock(() => undefined),
handlerFailed: mock(() => undefined),
@@ -77,13 +84,17 @@ function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {
async () => undefined
);
const sleep = mock(async (_ms: number) => undefined);
- const flushBufferedEvents = mock<
- EventsBatchHandlerDeps['flushBufferedEvents']
- >(async () => undefined);
+ // The gate a durability window closes with. The tests drive it directly;
+ // `openDurabilityWindow` below hands the same one to every batch, which is
+ // also what makes "opened before the first handler" assertable.
+ const flushBufferedEvents =
+ overrides.flushBufferedEvents ?? mock(async (): Promise<void> => undefined);
+ const openDurabilityWindow = mock(() => flushBufferedEvents);
+ const { flushBufferedEvents: _gate, ...depOverrides } = overrides;
const deps: EventsBatchHandlerDeps = {
handleEvent,
sendToDeadLetter,
- flushBufferedEvents,
+ openDurabilityWindow,
logger: {
info: mock(() => undefined),
warn: mock(() => undefined),
@@ -95,7 +106,7 @@ function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {
initialRetryMs: 10,
maxRetryMs: 25,
sleep,
- ...overrides,
+ ...depOverrides,
metrics: { ...metrics, ...overrides.metrics },
};
return {
@@ -104,8 +115,8 @@ function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {
handleEvent,
sendToDeadLetter: (overrides.sendToDeadLetter ??
sendToDeadLetter) as typeof sendToDeadLetter,
- flushBufferedEvents: (overrides.flushBufferedEvents ??
- flushBufferedEvents) as typeof flushBufferedEvents,
+ flushBufferedEvents,
+ openDurabilityWindow,
sleep,
};
}
@@ -510,6 +521,130 @@ describe('durability before commit', () => {
expect(metrics.reprocessed).not.toHaveBeenCalled();
});
+ test('opens the durability window before the first handler buffers anything', async () => {
+ // The window's lower bound is what makes the gate answer for THIS batch:
+ // a failed write drops the events it knows the broker will redeliver, so
+ // "everything queued is settled" is no longer "my events are in Redis"
+ // (M18-007).
+ let handled = 0;
+ let handledWhenOpened = -1;
+ const { deps, openDurabilityWindow } = makeDeps({
+ handleEvent: mock(async () => {
+ handled += 1;
+ return undefined;
+ }),
+ });
+ openDurabilityWindow.mockImplementation(() => {
+ handledWhenOpened = handled;
+ return async () => undefined;
+ });
+ const handler = createEventsBatchHandler(deps);
+ const { payload } = makeBatch([message(0, 'a'), message(1, 'b')]);
+
+ await handler.eachBatch(payload);
+
+ expect(openDurabilityWindow).toHaveBeenCalledTimes(1);
+ expect(handledWhenOpened).toBe(0);
+ });
+
+ test('a failed flush resolves NOTHING, so a redelivery is guaranteed', async () => {
+ // Asserted directly rather than left to the drill. With the re-queue gone
+ // (M18-007) the only thing standing between a failed flush and a lost
+ // batch is that its offsets are never resolved — every message, not just
+ // the tail, and regardless of how many of them the handlers finished.
+ const { deps, handleEvent } = makeDeps({
+ flushBufferedEvents: mock(() =>
+ Promise.reject(new Error('redis is unreachable'))
+ ),
+ });
+ const handler = createEventsBatchHandler(deps);
+ const { payload, resolveOffset } = makeBatch([
+ message(0, 'a'),
+ message(1, 'b'),
+ message(2, 'a'),
+ ]);
+
+ await handler.eachBatch(payload);
+
+ expect(handleEvent).toHaveBeenCalledTimes(3);
+ expect(resolveOffset).not.toHaveBeenCalled();
+ });
+
+ test('backs off before the redelivery, and the wait grows and then caps', async () => {
+ // Observable as a growing delay, not as a fixed attempt count: the point
+ // is that a ~15s outage costs single-digit redeliveries instead of the
+ // ~162 laps drill 02's re-run measured at ~90ms each.
+ const OUTAGE_LAPS = 8;
+ const { deps, sleep } = makeDeps({
+ flushBufferedEvents: mock(() =>
+ Promise.reject(new Error('redis is unreachable'))
+ ),
+ initialRetryMs: 100,
+ maxRetryMs: 1000,
+ });
+ const handler = createEventsBatchHandler(deps);
+
+ for (let lap = 0; lap < OUTAGE_LAPS; lap++) {
+ const { payload, resolveOffset } = makeBatch([message(0, 'a')]);
+ await handler.eachBatch(payload);
+ expect(resolvedOffsets(resolveOffset)).toEqual([]);
+ }
+
+ const waits = sleep.mock.calls.map(([ms]) => ms as number);
+ expect(waits.length).toBe(OUTAGE_LAPS);
+ expect(waits[0]).toBe(100);
+ for (let lap = 1; lap < OUTAGE_LAPS; lap++) {
+ expect(waits[lap]).toBeGreaterThanOrEqual(waits[lap - 1]!);
+ }
+ // Grown, and bounded — `maxRetryMs` is the flaky-handler bound, not this
+ // one, and the cap stays far below the 30s session timeout.
+ expect(waits.at(-1)).toBeGreaterThan(waits[0]!);
+ expect(Math.max(...waits)).toBe(5000);
+ // Single digit for an outage of this length, against ~162 without it.
+ expect(waits.reduce((total, ms) => total + ms, 0)).toBeGreaterThan(15_000);
+ });
+
+ test('forgets the backoff once the batch is durable again', async () => {
+ let redisIsDown = true;
+ const { deps, sleep } = makeDeps({
+ flushBufferedEvents: mock(async () => {
+ if (redisIsDown) {
+ throw new Error('redis is unreachable');
+ }
+ }),
+ initialRetryMs: 100,
+ });
+ const handler = createEventsBatchHandler(deps);
+
+ for (let lap = 0; lap < 3; lap++) {
+ await handler.eachBatch(makeBatch([message(lap, 'a')]).payload);
+ }
+ redisIsDown = false;
+ await handler.eachBatch(makeBatch([message(3, 'a')]).payload);
+ redisIsDown = true;
+ await handler.eachBatch(makeBatch([message(4, 'a')]).payload);
+
+ const waits = sleep.mock.calls.map(([ms]) => ms as number);
+ expect(waits).toEqual([100, 200, 400, 100]);
+ });
+
+ test('does not hold the partition while the consumer is shutting down', async () => {
+ const { deps, sleep } = makeDeps({
+ flushBufferedEvents: mock(() =>
+ Promise.reject(new Error('redis is unreachable'))
+ ),
+ });
+ const handler = createEventsBatchHandler(deps);
+ const { payload, resolveOffset } = makeBatch([message(0, 'a')], {
+ isRunning: () => false,
+ });
+
+ await handler.eachBatch(payload);
+
+ expect(resolvedOffsets(resolveOffset)).toEqual([]);
+ expect(sleep).not.toHaveBeenCalled();
+ });
+
test('flushes what a shutdown-truncated batch handled, before resolving it', async () => {
// Drill 03's shape: SIGTERM lands mid-batch, `isRunning` goes false, and
// the handler resolves only the prefix it finished. That prefix must be
diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.ts
index 390e47bc6..ea6ec287f 100644
--- a/packages/core/src/modules/ingest/src/consumer.ts
+++ b/packages/core/src/modules/ingest/src/consumer.ts
@@ -37,6 +37,22 @@ const HEARTBEAT_EVERY = 16;
const RETRY_BACKOFF_FACTOR = 2;
+/**
+ * Ceiling on the wait a batch takes before it returns from a failed
+ * durability flush and lets the broker redeliver it.
+ *
+ * The wait itself is seeded from `initialRetryMs` and doubled per consecutive
+ * failure — the consumer's own retry convention — but it is capped HERE
+ * rather than at `maxRetryMs`, which is sized for a flaky handler (1s by
+ * default) and not for a dependency that is down. Without any cap a failed
+ * flush redelivers in ~90ms and spins: drill 02's re-run measured ~162 laps
+ * per message in one 96s outage, hammering Redis and Kafka at the worst
+ * possible moment. 5s keeps a ~15s outage to single-digit redeliveries and
+ * stays well below the 30s session timeout, so waiting cannot cost the
+ * consumer its group membership (M18-007).
+ */
+const DURABILITY_RETRY_MAX_MS = 5000;
+
/** Why a message was dead-lettered. A label value — keep the set small. */
export type DeadLetterReason = 'parse_error' | 'handler_error';
@@ -77,17 +93,23 @@ export interface EventsBatchHandlerDeps {
) => Promise<unknown>;
sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;
/**
- * Make everything the handlers buffered in process memory durable.
+ * Opens a durability window over the event buffer and returns the gate that
+ * closes it — `EventBuffer.openDurabilityWindow`.
+ *
+ * Opened BEFORE the batch's first handler runs, and awaited once every
+ * message has been handled and BEFORE the first `resolveOffset`, because
+ * kafkajs commits the resolved offsets as soon as `eachBatch` returns
+ * (`autoCommit` defaults to true and this consumer does not turn it off) —
+ * not at `consumer.stop()`.
*
- * Called once every message in the batch has been handled and BEFORE the
- * first `resolveOffset`, because kafkajs commits the resolved offsets as
- * soon as `eachBatch` returns (`autoCommit` defaults to true and this
- * consumer does not turn it off) — not at `consumer.stop()`. It MUST reject
- * when the write did not land: drill 03 lost 2 of 44,075 events on a
- * graceful restart because the offsets of events that existed only in the
- * event buffer's in-process array were committed anyway (M18-001).
+ * The gate MUST reject when the events THIS batch buffered are not in
+ * Redis: drill 03 lost 2 of 44,075 events on a graceful restart because the
+ * offsets of events that existed only in the event buffer's in-process array
+ * were committed anyway (M18-001). The window is what makes "this batch's"
+ * answerable at all, now that a failed write drops the events it knows the
+ * broker will redeliver instead of keeping them (M18-007).
*/
- flushBufferedEvents: () => Promise<void>;
+ openDurabilityWindow: () => () => Promise<void>;
logger: ConsumerLogger;
metrics: ConsumerMetrics;
onActivity: () => void;
@@ -237,15 +259,20 @@ export function createEventsBatchHandler(
return deadLetter(message, partition, 'handler_error', result.err);
};
+ // Consecutive durability failures per `topic-partition`, for the backoff
+ // below. Cleared the moment a batch's events are durable again.
+ const durabilityFailures = new Map<string, number>();
+
/**
* @returns false when the flush failed, which leaves the watermark where it
* was so the redelivered batch is not then reported as a duplicate.
*/
const bufferedEventsAreDurable = async (
- partition: number
+ partition: number,
+ closeDurabilityWindow: () => Promise<void>
): Promise<boolean> => {
try {
- await deps.flushBufferedEvents();
+ await closeDurabilityWindow();
return true;
} catch (err) {
deps.logger.error(
@@ -256,6 +283,31 @@ export function createEventsBatchHandler(
}
};
+ /**
+ * Hold the partition before returning, so the redelivery this batch just
+ * asked for does not arrive in ~90ms and fail the same way.
+ */
+ const waitForRedelivery = async (
+ partitionKey: string,
+ partition: number,
+ heartbeat: () => Promise<void>
+ ): Promise<void> => {
+ const failures = (durabilityFailures.get(partitionKey) ?? 0) + 1;
+ durabilityFailures.set(partitionKey, failures);
+ const delayMs = Math.min(
+ deps.initialRetryMs * RETRY_BACKOFF_FACTOR ** (failures - 1),
+ DURABILITY_RETRY_MAX_MS
+ );
+ // Heartbeat first: a member that is deliberately waiting must not look
+ // like one that has died.
+ await heartbeat();
+ deps.logger.warn(
+ { partition, consecutiveFailures: failures, delayMs },
+ 'waiting before letting the broker redeliver the batch'
+ );
+ await sleep(delayMs);
+ };
+
const eachBatch = async ({
batch,
resolveOffset,
@@ -268,6 +320,10 @@ export function createEventsBatchHandler(
}
const pk = `${batch.topic}-${batch.partition}`;
+ // Opened before the first handler buffers anything, so the gate answers
+ // for THIS batch's events and not for whatever else was pending when it
+ // finished (M18-007).
+ const closeDurabilityWindow = deps.openDurabilityWindow();
// Watermark from *previous* batches. Anything at or below this that we see
// now is a redelivery. Captured before this batch so intra-batch
// out-of-order processing (normal, see below) is never counted.
@@ -344,7 +400,10 @@ export function createEventsBatchHandler(
// offset resolved below is committed the instant this handler returns. If
// that push will not land, resolve NOTHING and let the broker redeliver
// the whole batch — a duplicate is recoverable, a loss is not.
- if (await bufferedEventsAreDurable(batch.partition)) {
+ if (
+ await bufferedEventsAreDurable(batch.partition, closeDurabilityWindow)
+ ) {
+ durabilityFailures.delete(pk);
// Resolve in strict ascending offset order, stopping at the first offset
// that did not finish (e.g. an isStale/isRunning early-return mid-batch).
// batch.messages is already ordered by offset.
@@ -357,13 +416,23 @@ export function createEventsBatchHandler(
newHWM = Math.max(newHWM, Number(m.offset));
}
resolvedHWM.set(pk, newHWM);
+ } else if (isRunning() && !isStale()) {
+ // Not while shutting down: the redelivery is the next process's problem
+ // and the wait would only eat the shutdown budget.
+ await waitForRedelivery(pk, batch.partition, heartbeat);
}
await heartbeat();
deps.onActivity();
};
- return { eachBatch, resetWatermarks: () => resolvedHWM.clear() };
+ return {
+ eachBatch,
+ resetWatermarks: () => {
+ resolvedHWM.clear();
+ durabilityFailures.clear();
+ },
+ };
}
/** The broker-facing half, supplied by whoever owns the kafkajs client. */
diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.ts
index c063fd15d..7dea1f7ff 100644
--- a/packages/core/test/request-id-end-to-end.test.ts
+++ b/packages/core/test/request-id-end-to-end.test.ts
@@ -532,8 +532,15 @@ function ingestDeps() {
const producers = createRecordingProducers(queues);
const eventsBuffered: { id: string }[] = [];
+ const eventBuffer = {
+ add: (event: { id: string }) => eventsBuffered.push(event),
+ // The consumer scope reaches the buffer through this view (M18-007); it
+ // must still be the same buffer underneath.
+ asRedeliverable: () => eventBuffer,
+ };
+
const buffers = {
- event: { add: (event: { id: string }) => eventsBuffered.push(event) },
+ event: eventBuffer,
profile: { add: () => Promise.resolve() },
session: {
getExistingSession: () => Promise.resolve(null),