OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M18-007): Stop the redelivery amplification: drop the re-queue that M18-001 made double-counting, and add backoff so a failed flush cannot spin
ralph22f925693703
diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex a0f27d8be..efb5f9fa0 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -271,10 +271,11 @@ async function startIngestConsumer(}),sendToDeadLetter: (message) =>produceDeadLetterEvent(config.core, message),- // Runs before the batch's offsets are resolved, and THROWS if the rpush- // did not land. `pendingEvents` is the only in-process buffer state in- // the tree — every other buffer writes to Redis inside `add()`.- flushBufferedEvents: () => deps.buffers.event.flushPendingOrThrow(),+ // Opened before the batch's first handler and closed before its first+ // resolved offset; the gate THROWS if the rpush did not land.+ // `pendingEvents` is the only in-process buffer state in the tree —+ // every other buffer writes to Redis inside `add()`.+ openDurabilityWindow: () => deps.buffers.event.openDurabilityWindow(),logger,metrics: ingestConsumerMetrics,onActivity: markEventsActivity,diff --git a/packages/core/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.tsindex feee6cb23..2abb7ec7b 100644--- a/packages/core/src/buffers/event-buffer.test.ts+++ b/packages/core/src/buffers/event-buffer.test.ts@@ -539,6 +539,110 @@ describe('EventBuffer', () => {});});+ // M18-007 / drill 02 re-run: a failed flush leaves the batch's Kafka offsets+ // unresolved (M18-001) and the fail-fast cache client (M18-003) brings the+ // redelivery back in ~90ms. Re-queueing the failed events then holds one+ // copy per lap — 4,759 ClickHouse rows for 230 events, measured.+ describe('events a redelivery will produce again', () => {+ const REDELIVERY_LAPS = 5;++ /** One Kafka message's event. A redelivery builds an equal, NEW object. */+ const kafkaEvent = (name = 'redelivered') =>+ ({+ project_id: 'p18',+ name,+ created_at: '2026-09-13 20:50:31',+ }) as any;++ const queuedNames = async (): Promise<string[]> =>+ (await redis.lrange('event_buffer:queue', 0, -1)).map(+ (row) => (JSON.parse(row) as { name: string }).name+ );++ it('keeps ONE copy when the same message is redelivered through a whole outage', async () => {+ await redis.set('event_buffer:queue', 'not a list');++ for (let lap = 0; lap < REDELIVERY_LAPS; lap++) {+ const isDurable = eventBuffer.openDurabilityWindow();+ eventBuffer.addRedeliverable(kafkaEvent());+ await expect(isDurable()).rejects.toThrow('WRONGTYPE');+ // Nothing is held for a later retry: Kafka owns that now.+ expect(eventBuffer.getPendingLocalCount()).toBe(0);+ }++ await redis.del('event_buffer:queue');+ const isDurable = eventBuffer.openDurabilityWindow();+ eventBuffer.addRedeliverable(kafkaEvent());+ await isDurable();++ expect(await eventBuffer.getBufferSize()).toBe(1);+ });++ it('still holds the events of a producer nothing redelivers', async () => {+ await redis.set('event_buffer:queue', 'not a list');++ const isDurable = eventBuffer.openDurabilityWindow();+ eventBuffer.addRedeliverable(kafkaEvent('from-the-consumer'));+ // The session-end job's write: a BullMQ retry re-runs a job whose Redis+ // `SET NX` claim is already taken, so this copy is the only one there is.+ eventBuffer.add(kafkaEvent('from-the-session-end-job'));++ await expect(isDurable()).rejects.toThrow('WRONGTYPE');+ expect(eventBuffer.getPendingLocalCount()).toBe(1);++ await redis.del('event_buffer:queue');+ await eventBuffer.flush();++ expect(await queuedNames()).toEqual(['from-the-session-end-job']);+ });++ it('rejects the window whose events a write it never awaited dropped', async () => {+ // The load-bearing case. The micro-batch timer can start — and fail —+ // the write while the batch is still handling messages, so by the time+ // the gate is asked there is nothing pending and Redis may be healthy+ // again. Answering "everything queued is settled" would commit the+ // batch and lose it for good.+ await redis.set('event_buffer:queue', 'not a list');++ const isDurable = eventBuffer.openDurabilityWindow();+ eventBuffer.addRedeliverable(kafkaEvent());+ await eventBuffer.flush();+ expect(eventBuffer.getPendingLocalCount()).toBe(0);++ await redis.del('event_buffer:queue');++ await expect(isDurable()).rejects.toThrow('WRONGTYPE');+ expect(await eventBuffer.getBufferSize()).toBe(0);+ });++ it('does not poison a window opened after the drop', async () => {+ await redis.set('event_buffer:queue', 'not a list');+ const doomed = eventBuffer.openDurabilityWindow();+ eventBuffer.addRedeliverable(kafkaEvent('doomed'));+ await expect(doomed()).rejects.toThrow('WRONGTYPE');++ await redis.del('event_buffer:queue');+ const healthy = eventBuffer.openDurabilityWindow();+ eventBuffer.addRedeliverable(kafkaEvent('healthy'));+ await healthy();++ expect(await queuedNames()).toEqual(['healthy']);+ });++ it("routes the consumer scope's add() through addRedeliverable", async () => {+ await redis.set('event_buffer:queue', 'not a list');+ const consumerView = eventBuffer.asRedeliverable();++ const isDurable = eventBuffer.openDurabilityWindow();+ consumerView.add(kafkaEvent());+ await expect(isDurable()).rejects.toThrow('WRONGTYPE');++ // Same buffer underneath, and the event was dropped rather than held.+ expect(consumerView.getPendingLocalCount()).toBe(0);+ expect(eventBuffer.getPendingLocalCount()).toBe(0);+ });+ });+it('retains events in queue when ClickHouse insert fails', async () => {eventBuffer.add({project_id: 'p12',diff --git a/packages/core/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.tsindex 7636bf1b5..6bc29589f 100644--- a/packages/core/src/buffers/event-buffer.ts+++ b/packages/core/src/buffers/event-buffer.ts@@ -98,6 +98,28 @@ export class EventBuffer extends BaseBuffer {/** Tracks consecutive flush failures for observability; reset on success. */private flushRetryCount = 0;+ /**+ * Events whose producer re-produces them when this buffer reports a failed+ * flush. They are DROPPED by a failed write instead of being re-queued —+ * see `addRedeliverable`.+ */+ private readonly redeliverableEvents = new WeakSet<IClickhouseEvent>();++ /**+ * The highest sequence a failed write dropped, and the failure it dropped+ * it with. `lastDurableSeq` answers "is everything up to here in Redis?",+ * which stopped being the same question once a write could drop events: a+ * later write can carry the watermark past a sequence that was dropped+ * rather than written. A durability window (`openDurabilityWindow`) compares+ * against this so the producer that owns the dropped events is the one told+ * about it.+ */+ private droppedThroughSeq = 0;+ private lastDropFailure: unknown = null;++ /** Built once, on first use; see `asRedeliverable`. */+ private redeliverableView: EventBuffer | null = null;+private queueKey = 'event_buffer:queue';constructor(deps: BufferDeps) {@@ -167,7 +189,9 @@ export class EventBuffer extends BaseBuffer {* duplicate is recoverable, a dropped `pendingEvents` array is not* (M18-001, drill 03).*/- public async flushPendingOrThrow(): Promise<void> {+ public async flushPendingOrThrow(+ sinceSeq: number = this.droppedThroughSeq+ ): Promise<void> {const target = this.lastQueuedSeq;// Not a retry loop: each pass either waits out a write that is already@@ -180,10 +204,11 @@ export class EventBuffer extends BaseBuffer {if (inFlight === null) {if (this.pendingEvents.length === 0) {- // Nothing pending and nothing in flight, so every sequence ever- // queued has been written. Unreachable unless the watermark and the- // array disagree; returning beats spinning.- return;+ // Nothing pending and nothing in flight: every sequence queued is+ // SETTLED — written, or dropped for the producer that owns it to+ // produce again. Settled is not durable, which the check below is+ // what separates.+ break;}this.throwIfFailed(await this.flushLocalBuffer());continue;@@ -196,6 +221,72 @@ export class EventBuffer extends BaseBuffer {this.throwIfFailed(failure);}}++ // A write that dropped events did not make them durable, and the watermark+ // alone cannot say so: it moves on with the NEXT write, which carries none+ // of them. Without this a batch whose events were dropped by a micro-batch+ // timer it never awaited would be told they are safe, commit its offsets,+ // and lose them for good (M18-007).+ if (this.droppedThroughSeq > sinceSeq) {+ this.throwIfFailed(this.lastDropFailure);+ }+ }++ /**+ * Buffer an event whose producer produces it AGAIN when this buffer reports+ * a failed flush — today the Kafka consumer, whose batch is left+ * uncommitted and redelivered (`modules/ingest/src/consumer.ts`).+ *+ * Such an event is not re-queued by a failed write. Re-queueing it was+ * right while the buffer owned the retry, and became harmful the moment+ * Kafka took that ownership over (M18-001): the redelivery buffers a fresh+ * copy, so the old one is a SECOND copy of one event, and every further+ * redelivery adds another. Drill 02's re-run measured 4,759 ClickHouse rows+ * for 230 events that way (M18-007).+ *+ * Producers Kafka does not redeliver — the session-end job, whose Redis+ * `SET NX` claim makes a job retry a no-op — keep the safety net by using+ * plain `add()`.+ */+ public addRedeliverable(event: IClickhouseEvent): void {+ this.redeliverableEvents.add(event);+ this.add(event);+ }++ /**+ * A view of this buffer whose `add()` is `addRedeliverable()`; everything+ * else on it is this buffer's own.+ *+ * The ownership has to ride in on the SCOPE because `createEvent()` reaches+ * the buffer as `deps.buffers.event` and takes no ownership argument — so+ * the consumer hands its handlers a scope carrying this view+ * (`modules/ingest/src/consumer-handler.ts`) and every other transport keeps+ * the buffer itself.+ */+ public asRedeliverable(): EventBuffer {+ this.redeliverableView ??= new Proxy(this, {+ get: (target, property) =>+ property === 'add'+ ? (event: IClickhouseEvent) => target.addRedeliverable(event)+ : Reflect.get(target, property, target),+ });+ return this.redeliverableView;+ }++ /**+ * Opens a durability window and returns the gate that closes it: a promise+ * that resolves once everything buffered after this call is in Redis, and+ * rejects when any of it was dropped instead.+ *+ * The LOWER bound is the point. A caller that asks "is everything queued in+ * Redis?" cannot tell its own dropped events from another producer's, and a+ * failed write can drop events the caller never awaited. Opening the window+ * before the first event is buffered is what makes the answer that caller's+ * own (M18-007).+ */+ public openDurabilityWindow(): () => Promise<void> {+ const sinceSeq = this.lastQueuedSeq;+ return () => this.flushPendingOrThrow(sinceSeq);}private throwIfFailed(failure: unknown): void {@@ -274,14 +365,30 @@ export class EventBuffer extends BaseBuffer {this.flushRetryCount = 0;return null;} catch (error) {- // Re-queue failed events at the front to preserve order and avoid data loss- this.pendingEvents = eventsToFlush.concat(this.pendingEvents);+ // Re-queue only the events nothing else will produce again, at the front+ // to preserve order. A redeliverable event is already on its way back —+ // keeping it here would hold a second copy of it until Redis returns,+ // and one more for every redelivery in between (see `addRedeliverable`).+ const orphaned = eventsToFlush.filter(+ (event) => !this.redeliverableEvents.has(event)+ );+ const droppedCount = eventsToFlush.length - orphaned.length;+ if (droppedCount > 0) {+ // `coversSeq` over-states which sequences were dropped when the write+ // mixed the two kinds. Erring high costs an extra redelivery for a+ // window that straddled the failure; erring low would cost the events.+ this.droppedThroughSeq = Math.max(this.droppedThroughSeq, coversSeq);+ this.lastDropFailure = error;+ }+ this.pendingEvents = orphaned.concat(this.pendingEvents);this.flushRetryCount += 1;this.logger.warn({err: error,eventCount: eventsToFlush.length,+ requeuedCount: orphaned.length,+ droppedForRedeliveryCount: droppedCount,flushRetryCount: this.flushRetryCount,},'Failed to flush local buffer to Redis; events re-queued'diff --git a/packages/core/src/modules/ingest/src/consumer-handler.ts b/packages/core/src/modules/ingest/src/consumer-handler.tsindex 37dd152f3..29c118d75 100644--- a/packages/core/src/modules/ingest/src/consumer-handler.ts+++ b/packages/core/src/modules/ingest/src/consumer-handler.ts@@ -29,9 +29,20 @@ export function createIncomingEventHandler(deps: AppDeps,bindings: IncomingEventBindings) {+ // Every event this handler buffers is produced again by a redelivery when+ // the batch's durability gate fails, so the event buffer must not keep a+ // re-queued copy of it too — `EventBuffer.addRedeliverable`. `createEvent()`+ // reaches the buffer as `deps.buffers.event` and takes no ownership+ // argument, so the ownership rides in on the scope. Built once, here, not+ // per message: the hot path gets no work it did not already have.+ const consumerDeps: AppDeps = {+ ...deps,+ buffers: { ...deps.buffers, event: deps.buffers.event.asRedeliverable() },+ };+return (payload: IncomingEventPayload, meta: IncomingEventDelivery) => {const requestId = incomingEventRequestId(payload);- const ctx = createCtx(deps, {+ const ctx = createCtx(consumerDeps, {requestId,logger: deps.logger.child({ [REQUEST_ID_LOG_FIELD]: requestId }),});diff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.tsindex b2f9a42f6..8f1d3e26e 100644--- a/packages/core/src/modules/ingest/src/consumer.test.ts+++ b/packages/core/src/modules/ingest/src/consumer.test.ts@@ -63,7 +63,14 @@ interface MockCalls {const resolvedOffsets = (resolveOffset: MockCalls): string[] =>resolveOffset.mock.calls.map(([offset]) => offset as string);-function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {+type DepOverrides = Partial<+ Omit<EventsBatchHandlerDeps, 'openDurabilityWindow'>+> & {+ /** Sugar: the gate every window in this test closes with. */+ flushBufferedEvents?: () => Promise<void>;+};++function makeDeps(overrides: DepOverrides = {}) {const metrics: EventsBatchHandlerDeps['metrics'] = {reprocessed: mock(() => undefined),handlerFailed: mock(() => undefined),@@ -77,13 +84,17 @@ function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {async () => undefined);const sleep = mock(async (_ms: number) => undefined);- const flushBufferedEvents = mock<- EventsBatchHandlerDeps['flushBufferedEvents']- >(async () => undefined);+ // The gate a durability window closes with. The tests drive it directly;+ // `openDurabilityWindow` below hands the same one to every batch, which is+ // also what makes "opened before the first handler" assertable.+ const flushBufferedEvents =+ overrides.flushBufferedEvents ?? mock(async (): Promise<void> => undefined);+ const openDurabilityWindow = mock(() => flushBufferedEvents);+ const { flushBufferedEvents: _gate, ...depOverrides } = overrides;const deps: EventsBatchHandlerDeps = {handleEvent,sendToDeadLetter,- flushBufferedEvents,+ openDurabilityWindow,logger: {info: mock(() => undefined),warn: mock(() => undefined),@@ -95,7 +106,7 @@ function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {initialRetryMs: 10,maxRetryMs: 25,sleep,- ...overrides,+ ...depOverrides,metrics: { ...metrics, ...overrides.metrics },};return {@@ -104,8 +115,8 @@ function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {handleEvent,sendToDeadLetter: (overrides.sendToDeadLetter ??sendToDeadLetter) as typeof sendToDeadLetter,- flushBufferedEvents: (overrides.flushBufferedEvents ??- flushBufferedEvents) as typeof flushBufferedEvents,+ flushBufferedEvents,+ openDurabilityWindow,sleep,};}@@ -510,6 +521,130 @@ describe('durability before commit', () => {expect(metrics.reprocessed).not.toHaveBeenCalled();});+ test('opens the durability window before the first handler buffers anything', async () => {+ // The window's lower bound is what makes the gate answer for THIS batch:+ // a failed write drops the events it knows the broker will redeliver, so+ // "everything queued is settled" is no longer "my events are in Redis"+ // (M18-007).+ let handled = 0;+ let handledWhenOpened = -1;+ const { deps, openDurabilityWindow } = makeDeps({+ handleEvent: mock(async () => {+ handled += 1;+ return undefined;+ }),+ });+ openDurabilityWindow.mockImplementation(() => {+ handledWhenOpened = handled;+ return async () => undefined;+ });+ const handler = createEventsBatchHandler(deps);+ const { payload } = makeBatch([message(0, 'a'), message(1, 'b')]);++ await handler.eachBatch(payload);++ expect(openDurabilityWindow).toHaveBeenCalledTimes(1);+ expect(handledWhenOpened).toBe(0);+ });++ test('a failed flush resolves NOTHING, so a redelivery is guaranteed', async () => {+ // Asserted directly rather than left to the drill. With the re-queue gone+ // (M18-007) the only thing standing between a failed flush and a lost+ // batch is that its offsets are never resolved — every message, not just+ // the tail, and regardless of how many of them the handlers finished.+ const { deps, handleEvent } = makeDeps({+ flushBufferedEvents: mock(() =>+ Promise.reject(new Error('redis is unreachable'))+ ),+ });+ const handler = createEventsBatchHandler(deps);+ const { payload, resolveOffset } = makeBatch([+ message(0, 'a'),+ message(1, 'b'),+ message(2, 'a'),+ ]);++ await handler.eachBatch(payload);++ expect(handleEvent).toHaveBeenCalledTimes(3);+ expect(resolveOffset).not.toHaveBeenCalled();+ });++ test('backs off before the redelivery, and the wait grows and then caps', async () => {+ // Observable as a growing delay, not as a fixed attempt count: the point+ // is that a ~15s outage costs single-digit redeliveries instead of the+ // ~162 laps drill 02's re-run measured at ~90ms each.+ const OUTAGE_LAPS = 8;+ const { deps, sleep } = makeDeps({+ flushBufferedEvents: mock(() =>+ Promise.reject(new Error('redis is unreachable'))+ ),+ initialRetryMs: 100,+ maxRetryMs: 1000,+ });+ const handler = createEventsBatchHandler(deps);++ for (let lap = 0; lap < OUTAGE_LAPS; lap++) {+ const { payload, resolveOffset } = makeBatch([message(0, 'a')]);+ await handler.eachBatch(payload);+ expect(resolvedOffsets(resolveOffset)).toEqual([]);+ }++ const waits = sleep.mock.calls.map(([ms]) => ms as number);+ expect(waits.length).toBe(OUTAGE_LAPS);+ expect(waits[0]).toBe(100);+ for (let lap = 1; lap < OUTAGE_LAPS; lap++) {+ expect(waits[lap]).toBeGreaterThanOrEqual(waits[lap - 1]!);+ }+ // Grown, and bounded — `maxRetryMs` is the flaky-handler bound, not this+ // one, and the cap stays far below the 30s session timeout.+ expect(waits.at(-1)).toBeGreaterThan(waits[0]!);+ expect(Math.max(...waits)).toBe(5000);+ // Single digit for an outage of this length, against ~162 without it.+ expect(waits.reduce((total, ms) => total + ms, 0)).toBeGreaterThan(15_000);+ });++ test('forgets the backoff once the batch is durable again', async () => {+ let redisIsDown = true;+ const { deps, sleep } = makeDeps({+ flushBufferedEvents: mock(async () => {+ if (redisIsDown) {+ throw new Error('redis is unreachable');+ }+ }),+ initialRetryMs: 100,+ });+ const handler = createEventsBatchHandler(deps);++ for (let lap = 0; lap < 3; lap++) {+ await handler.eachBatch(makeBatch([message(lap, 'a')]).payload);+ }+ redisIsDown = false;+ await handler.eachBatch(makeBatch([message(3, 'a')]).payload);+ redisIsDown = true;+ await handler.eachBatch(makeBatch([message(4, 'a')]).payload);++ const waits = sleep.mock.calls.map(([ms]) => ms as number);+ expect(waits).toEqual([100, 200, 400, 100]);+ });++ test('does not hold the partition while the consumer is shutting down', async () => {+ const { deps, sleep } = makeDeps({+ flushBufferedEvents: mock(() =>+ Promise.reject(new Error('redis is unreachable'))+ ),+ });+ const handler = createEventsBatchHandler(deps);+ const { payload, resolveOffset } = makeBatch([message(0, 'a')], {+ isRunning: () => false,+ });++ await handler.eachBatch(payload);++ expect(resolvedOffsets(resolveOffset)).toEqual([]);+ expect(sleep).not.toHaveBeenCalled();+ });+test('flushes what a shutdown-truncated batch handled, before resolving it', async () => {// Drill 03's shape: SIGTERM lands mid-batch, `isRunning` goes false, and// the handler resolves only the prefix it finished. That prefix must bediff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.tsindex 390e47bc6..ea6ec287f 100644--- a/packages/core/src/modules/ingest/src/consumer.ts+++ b/packages/core/src/modules/ingest/src/consumer.ts@@ -37,6 +37,22 @@ const HEARTBEAT_EVERY = 16;const RETRY_BACKOFF_FACTOR = 2;+/**+ * Ceiling on the wait a batch takes before it returns from a failed+ * durability flush and lets the broker redeliver it.+ *+ * The wait itself is seeded from `initialRetryMs` and doubled per consecutive+ * failure — the consumer's own retry convention — but it is capped HERE+ * rather than at `maxRetryMs`, which is sized for a flaky handler (1s by+ * default) and not for a dependency that is down. Without any cap a failed+ * flush redelivers in ~90ms and spins: drill 02's re-run measured ~162 laps+ * per message in one 96s outage, hammering Redis and Kafka at the worst+ * possible moment. 5s keeps a ~15s outage to single-digit redeliveries and+ * stays well below the 30s session timeout, so waiting cannot cost the+ * consumer its group membership (M18-007).+ */+const DURABILITY_RETRY_MAX_MS = 5000;+/** Why a message was dead-lettered. A label value — keep the set small. */export type DeadLetterReason = 'parse_error' | 'handler_error';@@ -77,17 +93,23 @@ export interface EventsBatchHandlerDeps {) => Promise<unknown>;sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;/**- * Make everything the handlers buffered in process memory durable.+ * Opens a durability window over the event buffer and returns the gate that+ * closes it — `EventBuffer.openDurabilityWindow`.+ *+ * Opened BEFORE the batch's first handler runs, and awaited once every+ * message has been handled and BEFORE the first `resolveOffset`, because+ * kafkajs commits the resolved offsets as soon as `eachBatch` returns+ * (`autoCommit` defaults to true and this consumer does not turn it off) —+ * not at `consumer.stop()`.*- * Called once every message in the batch has been handled and BEFORE the- * first `resolveOffset`, because kafkajs commits the resolved offsets as- * soon as `eachBatch` returns (`autoCommit` defaults to true and this- * consumer does not turn it off) — not at `consumer.stop()`. It MUST reject- * when the write did not land: drill 03 lost 2 of 44,075 events on a- * graceful restart because the offsets of events that existed only in the- * event buffer's in-process array were committed anyway (M18-001).+ * The gate MUST reject when the events THIS batch buffered are not in+ * Redis: drill 03 lost 2 of 44,075 events on a graceful restart because the+ * offsets of events that existed only in the event buffer's in-process array+ * were committed anyway (M18-001). The window is what makes "this batch's"+ * answerable at all, now that a failed write drops the events it knows the+ * broker will redeliver instead of keeping them (M18-007).*/- flushBufferedEvents: () => Promise<void>;+ openDurabilityWindow: () => () => Promise<void>;logger: ConsumerLogger;metrics: ConsumerMetrics;onActivity: () => void;@@ -237,15 +259,20 @@ export function createEventsBatchHandler(return deadLetter(message, partition, 'handler_error', result.err);};+ // Consecutive durability failures per `topic-partition`, for the backoff+ // below. Cleared the moment a batch's events are durable again.+ const durabilityFailures = new Map<string, number>();+/*** @returns false when the flush failed, which leaves the watermark where it* was so the redelivered batch is not then reported as a duplicate.*/const bufferedEventsAreDurable = async (- partition: number+ partition: number,+ closeDurabilityWindow: () => Promise<void>): Promise<boolean> => {try {- await deps.flushBufferedEvents();+ await closeDurabilityWindow();return true;} catch (err) {deps.logger.error(@@ -256,6 +283,31 @@ export function createEventsBatchHandler(}};+ /**+ * Hold the partition before returning, so the redelivery this batch just+ * asked for does not arrive in ~90ms and fail the same way.+ */+ const waitForRedelivery = async (+ partitionKey: string,+ partition: number,+ heartbeat: () => Promise<void>+ ): Promise<void> => {+ const failures = (durabilityFailures.get(partitionKey) ?? 0) + 1;+ durabilityFailures.set(partitionKey, failures);+ const delayMs = Math.min(+ deps.initialRetryMs * RETRY_BACKOFF_FACTOR ** (failures - 1),+ DURABILITY_RETRY_MAX_MS+ );+ // Heartbeat first: a member that is deliberately waiting must not look+ // like one that has died.+ await heartbeat();+ deps.logger.warn(+ { partition, consecutiveFailures: failures, delayMs },+ 'waiting before letting the broker redeliver the batch'+ );+ await sleep(delayMs);+ };+const eachBatch = async ({batch,resolveOffset,@@ -268,6 +320,10 @@ export function createEventsBatchHandler(}const pk = `${batch.topic}-${batch.partition}`;+ // Opened before the first handler buffers anything, so the gate answers+ // for THIS batch's events and not for whatever else was pending when it+ // finished (M18-007).+ const closeDurabilityWindow = deps.openDurabilityWindow();// Watermark from *previous* batches. Anything at or below this that we see// now is a redelivery. Captured before this batch so intra-batch// out-of-order processing (normal, see below) is never counted.@@ -344,7 +400,10 @@ export function createEventsBatchHandler(// offset resolved below is committed the instant this handler returns. If// that push will not land, resolve NOTHING and let the broker redeliver// the whole batch — a duplicate is recoverable, a loss is not.- if (await bufferedEventsAreDurable(batch.partition)) {+ if (+ await bufferedEventsAreDurable(batch.partition, closeDurabilityWindow)+ ) {+ durabilityFailures.delete(pk);// Resolve in strict ascending offset order, stopping at the first offset// that did not finish (e.g. an isStale/isRunning early-return mid-batch).// batch.messages is already ordered by offset.@@ -357,13 +416,23 @@ export function createEventsBatchHandler(newHWM = Math.max(newHWM, Number(m.offset));}resolvedHWM.set(pk, newHWM);+ } else if (isRunning() && !isStale()) {+ // Not while shutting down: the redelivery is the next process's problem+ // and the wait would only eat the shutdown budget.+ await waitForRedelivery(pk, batch.partition, heartbeat);}await heartbeat();deps.onActivity();};- return { eachBatch, resetWatermarks: () => resolvedHWM.clear() };+ return {+ eachBatch,+ resetWatermarks: () => {+ resolvedHWM.clear();+ durabilityFailures.clear();+ },+ };}/** The broker-facing half, supplied by whoever owns the kafkajs client. */diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.tsindex c063fd15d..7dea1f7ff 100644--- a/packages/core/test/request-id-end-to-end.test.ts+++ b/packages/core/test/request-id-end-to-end.test.ts@@ -532,8 +532,15 @@ function ingestDeps() {const producers = createRecordingProducers(queues);const eventsBuffered: { id: string }[] = [];+ const eventBuffer = {+ add: (event: { id: string }) => eventsBuffered.push(event),+ // The consumer scope reaches the buffer through this view (M18-007); it+ // must still be the same buffer underneath.+ asRedeliverable: () => eventBuffer,+ };+const buffers = {- event: { add: (event: { id: string }) => eventsBuffered.push(event) },+ event: eventBuffer,profile: { add: () => Promise.resolve() },session: {getExistingSession: () => Promise.resolve(null),