:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { CODES, Message, KafkaConsumer as RdKafkaConsumer } from 'node-rdkafka'
import { ImageFetchBatchJoiner } from '../../../ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetch-batch-joiner'
import { captureException } from '../../utils/posthog'
import { delay } from '../../utils/utils'
import { KafkaConsumerV2 } from './consumer-v2'
jest.mock('../admin', () => ({ ensureTopicExists: jest.fn().mockResolvedValue(undefined) }))
// Spy on captureException to assert that the IDLE-keepalive invariant fires when violated.
jest.mock('../../utils/posthog', () => ({
captureException: jest.fn(),
}))
jest.mock('node-rdkafka', () => ({
KafkaConsumer: jest.fn(),
CODES: {
ERRORS: {
ERR__REVOKE_PARTITIONS: 'ERR__REVOKE_PARTITIONS',
ERR__ASSIGN_PARTITIONS: 'ERR__ASSIGN_PARTITIONS',
},
},
}))
const createMessage = (m: Partial<Message> = {}): Message => ({
value: Buffer.from('test-value'),
key: Buffer.from('test-key'),
offset: 1,
partition: 0,
topic: 'test-topic',
size: 10,
...m,
})
jest.setTimeout(15000)
const activePromiseResolvers: Array<() => void> = []
const triggerablePromise = <T = unknown>(): {
promise: Promise<T>
resolve: (value?: T) => void
reject: (reason?: unknown) => void
} => {
const result = {
promise: null as unknown as Promise<T>,
resolve: () => {},
reject: () => {},
} as {
promise: Promise<T>
resolve: (value?: T) => void
reject: (reason?: unknown) => void
}
result.promise = new Promise((resolve, reject) => {
const wrappedResolve = (value?: T) => {
const i = activePromiseResolvers.indexOf(wrappedResolve)
if (i > -1) {
activePromiseResolvers.splice(i, 1)
}
resolve(value as T)
}
result.resolve = wrappedResolve
result.reject = (reason?: unknown) => {
const i = activePromiseResolvers.indexOf(wrappedResolve)
if (i > -1) {
activePromiseResolvers.splice(i, 1)
}
reject(reason)
}
activePromiseResolvers.push(wrappedResolve)
})
return result
}
describe('KafkaConsumerV2', () => {
let consumer: KafkaConsumerV2
let mockRdKafka: jest.Mocked<RdKafkaConsumer>
let consumeCallback: ((error: Error | null, messages: Message[]) => void) | undefined
let registeredRebalanceCb: ((err: any, partitions: any[]) => void) | undefined
beforeEach(() => {
consumeCallback = undefined
registeredRebalanceCb = undefined
;(captureException as jest.Mock).mockClear()
const mockInstance: any = {
connect: jest.fn().mockImplementation((_: unknown, cb: (err: Error | null) => void) => cb(null)),
subscribe: jest.fn(),
consume: jest.fn().mockImplementation((_: unknown, cb: any) => {
consumeCallback = cb
}),
disconnect: jest.fn().mockImplementation((cb: (err: Error | null) => void) => cb(null)),
isConnected: jest.fn().mockReturnValue(true),
on: jest.fn(),
assignments: jest.fn().mockReturnValue([]),
offsetsStore: jest.fn(),
setDefaultConsumeTimeout: jest.fn(),
incrementalUnassign: jest.fn(),
incrementalAssign: jest.fn(),
assign: jest.fn(),
unassign: jest.fn(),
rebalanceProtocol: jest.fn().mockReturnValue('COOPERATIVE'),
}
// Capture the rebalance callback the consumer registers via rdKafkaConfig
const RdKafkaCtor = jest.mocked(require('node-rdkafka').KafkaConsumer)
RdKafkaCtor.mockImplementation((config: any) => {
registeredRebalanceCb = config.rebalance_cb
return mockInstance
})
consumer = new KafkaConsumerV2({ groupId: 'test-group', topic: 'test-topic' })
mockRdKafka = mockInstance
})
afterEach(async () => {
// Drop any in-flight tasks before disconnect to avoid 60s timeout waits.
if (consumer && (consumer as any).inFlight) {
;(consumer as any).inFlight = []
}
while (activePromiseResolvers.length > 0) {
activePromiseResolvers[0]()
}
if (consumer) {
const p = consumer.disconnect()
// Release any pending consume callback so the loop can exit.
if (consumeCallback) {
consumeCallback(null, [])
}
await p
}
})
/** Drive the consumer through ASSIGN so it transitions to CONSUMING. */
const startConsuming = async (
eachBatch: jest.Mock,
partitions = [{ topic: 'test-topic', partition: 0 }],
onPartitionsRevoked?: (assignments: { topic: string; partition: number }[]) => Promise<void>,
onOffsetsStored?: jest.Mock
) => {
await consumer.connect(eachBatch, onPartitionsRevoked, onOffsetsStored)
registeredRebalanceCb!({ code: CODES.ERRORS.ERR__ASSIGN_PARTITIONS } as any, partitions)
// The loop is currently inside the IDLE keepalive consume(1, cb). Release it with
// an empty batch so the loop processes ASSIGN and arms a fresh consume() in CONSUMING.
if (consumeCallback) {
const cb = consumeCallback
consumeCallback = undefined
cb(null, [])
}
await delay(5)
}
/** Release the in-flight consume(), if any, with empty messages so the loop can iterate. */
const releaseConsume = () => {
if (consumeCallback) {
const cb = consumeCallback
consumeCallback = undefined
cb(null, [])
}
}
const fireRevoke = (partitions = [{ topic: 'test-topic', partition: 0 }]) => {
registeredRebalanceCb!({ code: CODES.ERRORS.ERR__REVOKE_PARTITIONS } as any, partitions)
// librdkafka delivers REVOKE on the same thread as consume() returns; simulate that.
releaseConsume()
}
const fireAssign = (partitions = [{ topic: 'test-topic', partition: 0 }]) => {
registeredRebalanceCb!({ code: CODES.ERRORS.ERR__ASSIGN_PARTITIONS } as any, partitions)
releaseConsume()
}
/** Deliver a batch + a controllable backgroundTask. Resolves when the task is in-flight. */
const dispatchBatch = async (
eachBatch: jest.Mock,
messages: Message[],
backgroundTask?: Promise<unknown>
): Promise<void> => {
eachBatch.mockImplementationOnce(() =>
backgroundTask !== undefined ? Promise.resolve({ backgroundTask }) : Promise.resolve({})
)
consumeCallback!(null, messages)
await delay(2)
}
// auto.offset.reset is a topic-level librdkafka property — a value only in the global config
// is ignored, so the override must be mirrored into the topic config (2nd ctor arg).
it.each([
[
'mirrors an auto.offset.reset override into the topic config',
{ 'auto.offset.reset': 'latest' } as any,
'latest',
],
['defaults the topic-config auto.offset.reset to earliest with no override', undefined, 'earliest'],
])('%s', (_label, overrides, expected) => {
const RdKafkaCtor = jest.mocked(RdKafkaConsumer)
new KafkaConsumerV2({ groupId: 'g', topic: 't' }, overrides)
const topicConfig = RdKafkaCtor.mock.calls.at(-1)![1] as any
expect(topicConfig['auto.offset.reset']).toBe(expected)
})
it('Smoke: connect → consume → eachBatch → offsets stored', async () => {
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
consumeCallback!(null, [createMessage({ offset: 1, partition: 0 })])
await delay(5)
expect(eachBatch).toHaveBeenCalledWith([createMessage({ offset: 1, partition: 0 })])
expect(mockRdKafka.offsetsStore).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0, offset: 2 }])
})
it.each(['direct', 'image fetch joiner'])(
'bounds %s background work and holds later offsets until the oldest batch settles',
async (mode) => {
consumer = new KafkaConsumerV2({ groupId: 'test-group', topic: 'test-topic', maxBackgroundTasks: 2 })
const p1 = triggerablePromise<void>()
const p2 = triggerablePromise<void>()
const processBatch = jest.fn().mockReturnValueOnce(p1.promise).mockReturnValueOnce(p2.promise)
const joiner = new ImageFetchBatchJoiner(1, processBatch)
const eachBatch = jest.fn((messages: Message[]) =>
mode === 'direct'
? Promise.resolve({ backgroundTask: processBatch(messages) })
: joiner.handleBatch(messages)
)
const onOffsetsStored = jest.fn()
await startConsuming(eachBatch, undefined, undefined, onOffsetsStored)
consumeCallback!(null, [createMessage({ offset: 1, partition: 0 })])
await delay(5)
const pollsAfterFirstBatch = mockRdKafka.consume.mock.calls.length
consumeCallback!(null, [createMessage({ offset: 2, partition: 0 })])
await delay(5)
expect(processBatch).toHaveBeenCalledTimes(2)
expect(mockRdKafka.consume).toHaveBeenCalledTimes(pollsAfterFirstBatch)
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
p2.resolve()
await delay(5)
expect(mockRdKafka.consume).toHaveBeenCalledTimes(pollsAfterFirstBatch)
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
expect(onOffsetsStored).not.toHaveBeenCalled()
p1.resolve()
await delay(5)
expect(mockRdKafka.consume.mock.calls.length).toBeGreaterThan(pollsAfterFirstBatch)
expect(mockRdKafka.offsetsStore.mock.calls).toEqual([
[[{ topic: 'test-topic', partition: 0, offset: 2 }]],
[[{ topic: 'test-topic', partition: 0, offset: 3 }]],
])
expect(onOffsetsStored.mock.calls).toEqual(mockRdKafka.offsetsStore.mock.calls)
}
)
it('reports only the offsets that librdkafka accepted to store', async () => {
const eachBatch = jest.fn(() => Promise.resolve({}))
const onOffsetsStored = jest.fn()
await startConsuming(eachBatch, undefined, undefined, onOffsetsStored)
mockRdKafka.offsetsStore.mockImplementationOnce(() => {
throw new Error('Local: Erroneous state')
})
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], Promise.resolve())
await dispatchBatch(eachBatch, [createMessage({ offset: 2, partition: 0 })], Promise.resolve())
await delay(10)
expect(mockRdKafka.offsetsStore).toHaveBeenCalledTimes(2)
expect(onOffsetsStored.mock.calls).toEqual([[[{ topic: 'test-topic', partition: 0, offset: 3 }]]])
})
it('REVOKE drain: incrementalUnassign called only after every settled completes', async () => {
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const p1 = triggerablePromise()
const p2 = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], p1.promise)
await dispatchBatch(eachBatch, [createMessage({ offset: 2, partition: 0 })], p2.promise)
fireRevoke()
await delay(5)
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
p1.resolve()
await delay(5)
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
p2.resolve()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
})
it('H1 regression: task pushed during the rebalance window is awaited before unassign', async () => {
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const p1 = triggerablePromise()
const p2 = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], p1.promise)
await dispatchBatch(eachBatch, [createMessage({ offset: 2, partition: 0 })], p2.promise)
// Set up a third batch where eachBatch is in flight when the rebalance fires.
const p3 = triggerablePromise()
const eachBatchGate = triggerablePromise()
eachBatch.mockImplementationOnce(async () => {
await eachBatchGate.promise
return { backgroundTask: p3.promise }
})
consumeCallback!(null, [createMessage({ offset: 3, partition: 0 })])
await delay(2)
// Fire revoke while eachBatch is still awaiting the gate.
fireRevoke()
await delay(2)
// Release eachBatch — even though we're now DRAINING, the task is tracked and drainAll
// awaits its settled promise. (This is the H1 fix: tasks pushed during the rebalance
// window are NOT excluded from drain.)
eachBatchGate.resolve()
await delay(5)
// Validate the fix mechanism, not just the outcome: all 3 tasks are tracked in inFlight,
// including t3 which was dispatched AFTER REVOKE arrived.
expect((consumer as any).inFlight.length).toBe(3)
// Resolve t1 + t2 only. t3 still pending.
p1.resolve()
p2.resolve()
await delay(10)
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
// Resolving t3 unblocks drain.
p3.resolve()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
})
it('H2 regression: a task settling inside the drain stores its offsets, and a sticky reassign arms nothing extra', async () => {
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const slow = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], slow.promise)
// Fire revoke; drain begins.
fireRevoke()
await delay(2)
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
// Resolve slow. Drain completes, unassign fires synchronously inside the loop.
slow.resolve()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledTimes(1)
// The task finished inside the drain budget, while the partition was still assigned, so
// its offsets are stored (and committed on the unassign below) rather than discarded —
// discarding them is what made the next owner replay completed work. A task that misses
// the budget is fenced instead: see the "Partition fence" test.
expect(mockRdKafka.offsetsStore).toHaveBeenCalledWith([{ offset: 2, partition: 0, topic: 'test-topic' }])
// Snapshot offsetsStore + unassign call counts before the reassign so we can assert
// nothing further happens once we re-acquire the partition.
const offsetCallsBeforeReassign = mockRdKafka.offsetsStore.mock.calls.length
mockRdKafka.incrementalUnassign.mockClear()
// Now simulate cooperative-sticky reassign of the same partition.
fireAssign()
await delay(5)
expect(mockRdKafka.incrementalAssign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
// No deferred .then armed — no bogus unassign and no surprise offset writes against
// the just-reassigned partition.
await delay(20)
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
expect(mockRdKafka.offsetsStore.mock.calls.length).toBe(offsetCallsBeforeReassign)
})
it('store order survives an empty batch that lingers in inFlight: a later batch waits for the earlier storing one', async () => {
// Empty batches no longer chain, which is what stops them holding a backpressure slot.
// But an empty batch still enters inFlight when it carries a backgroundTask, and the CDP
// cyclotron worker returns one for an empty poll. Chaining a storing batch on the tail of
// inFlight would then let it inherit the empty batch's settle and store ahead of the
// earlier storing batch — the exact reordering the chain exists to prevent.
;(consumer as any).maxBackgroundTasks = 10
;(consumer as any).config.callEachBatchWhenEmpty = true
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const slowFirst = triggerablePromise()
const emptyBatchTask = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], slowFirst.promise)
await dispatchBatch(eachBatch, [], emptyBatchTask.promise)
await dispatchBatch(eachBatch, [createMessage({ offset: 2, partition: 0 })], undefined)
// The empty batch finishes first. The later batch must still be waiting on the first one.
emptyBatchTask.resolve()
await delay(20)
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
slowFirst.resolve()
await delay(20)
expect(mockRdKafka.offsetsStore.mock.calls.map((call) => call[0])).toEqual([
[{ topic: 'test-topic', partition: 0, offset: 2 }],
[{ topic: 'test-topic', partition: 0, offset: 3 }],
])
})
it('a failed batch blocks a later store even when an empty batch sits between them', async () => {
// The store chain is what keeps a later batch from advancing the committed offset past an
// earlier batch that failed. An empty poll (callEachBatchWhenEmpty, as the CDP cyclotron
// worker sets) still enters inFlight when it carries a backgroundTask, so excluding empty
// batches from the chain reopens that gap and offsets advance past failed work.
;(consumer as any).maxBackgroundTasks = 10
;(consumer as any).config.callEachBatchWhenEmpty = true
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const failing = triggerablePromise()
await dispatchBatch(eachBatch, [], failing.promise)
await dispatchBatch(eachBatch, [createMessage({ offset: 7, partition: 0 })], Promise.resolve())
await delay(10)
// The later batch must not have stored yet: the empty batch ahead of it has not settled.
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
failing.reject(new Error('empty batch work failed'))
await delay(30)
// And once that batch fails, the store is abandoned rather than committed.
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
})
it('the fence is per partition: a laggard task still stores the partition it kept, never the one it lost', async () => {
;(consumer as any).maxBackgroundTasks = 5
;(consumer as any).drainTimeoutMs = 30 // the task settles after the drain gives up
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch, [
{ topic: 'test-topic', partition: 0 },
{ topic: 'test-topic', partition: 1 },
])
const slow = triggerablePromise()
await dispatchBatch(
eachBatch,
[createMessage({ offset: 5, partition: 0 }), createMessage({ offset: 9, partition: 1 })],
slow.promise
)
// Only partition 0 is revoked; we keep partition 1.
fireRevoke([{ topic: 'test-topic', partition: 0 }])
await delay(60)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
slow.resolve()
await delay(20)
// Partition 1's progress is ours to commit — it never left. Partition 0's is not, because
// another member may own it now. A global fence would have dropped both, stalling
// partition 1's commit until some later revoke replayed everything since.
expect(mockRdKafka.offsetsStore).toHaveBeenCalledTimes(1)
expect(mockRdKafka.offsetsStore).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 1, offset: 10 }])
})
it('H3 regression: out-of-order task completion — drain awaits ALL settled before unassign', async () => {
// The v1 H3 race was: drain awaited t.promise (raw), so the late task's storeOffsets
// could fire AFTER incrementalUnassign. v2 fixes this two ways:
// (a) drain awaits the post-storeOffsets `settled` chain, not the raw task; and
// (b) the epoch bump after the drain fences anything that settles later, so a task
// that missed the budget can't store against a partition we gave up (see the
// "Partition fence" test).
//
// This test verifies (a) directly: with two tasks resolving out of order, drainAll awaits
// both settled callbacks before incrementalUnassign fires. If drain awaited only `raw`,
// it could fire before the second task settled.
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const p1 = triggerablePromise()
const p2 = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], p1.promise)
await dispatchBatch(eachBatch, [createMessage({ offset: 2, partition: 0 })], p2.promise)
fireRevoke()
await delay(2)
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
// Resolve out of order — t2 finishes first.
p2.resolve()
await delay(10)
// Drain still awaits t1's settled — unassign must NOT have fired yet.
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
// Now t1 settles → drain completes → unassign fires.
p1.resolve()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
})
it('Drain timeout: never-settling task is force-released after drainTimeoutMs', async () => {
;(consumer as any).drainTimeoutMs = 50
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const stuck = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], stuck.promise)
fireRevoke()
await delay(120)
// Drain timed out; loop proceeded with unassign.
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalled()
})
it('Revoke hook timeout: a hook outliving drainTimeoutMs is abandoned and unassign proceeds', async () => {
;(consumer as any).drainTimeoutMs = 50
const hookGate = triggerablePromise<void>()
const hook = jest.fn(() => hookGate.promise)
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch, [{ topic: 'test-topic', partition: 0 }], hook)
fireRevoke()
await delay(20)
// Within budget: the rebalance waits for the hook (flush-on-revoke).
expect(hook).toHaveBeenCalled()
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
await delay(100)
// Budget exceeded: the partitions are given up without waiting for the hook.
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
hookGate.resolve()
})
it('Revoke budget is total: a stuck drain leaves no extra budget for the hook', async () => {
;(consumer as any).drainTimeoutMs = 150
;(consumer as any).maxBackgroundTasks = 5
const hookGate = triggerablePromise<void>()
const hook = jest.fn(() => hookGate.promise)
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch, [{ topic: 'test-topic', partition: 0 }], hook)
// A never-settling task forces the drain to spend the entire budget.
const stuck = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], stuck.promise)
fireRevoke()
// The hook shares the drain's budget rather than getting its own: with the budget
// exhausted, it is abandoned immediately and the total hold stays ~one budget, not two.
await delay(220)
expect(hook).toHaveBeenCalled()
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalled()
hookGate.resolve()
stuck.resolve()
})
it('ASSIGN after REVOKE: state transitions back to CONSUMING and resumes fetching', async () => {
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
fireRevoke()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalled()
const callsBeforeAssign = mockRdKafka.consume.mock.calls.length
fireAssign([{ topic: 'test-topic', partition: 1 }])
await delay(10)
expect(mockRdKafka.incrementalAssign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 1 }])
// After ASSIGN, the loop transitions back to CONSUMING and immediately calls consume().
expect(mockRdKafka.consume.mock.calls.length).toBeGreaterThan(callsBeforeAssign)
})
it('REVOKE without follow-up ASSIGN: loop stays alive and keeps polling', async () => {
// Validates that if librdkafka revokes our partitions and never re-assigns (e.g. the
// group went down to one consumer), the loop doesn't wedge — it keeps calling
// consume() to drive heartbeats and the next rebalance.
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
fireRevoke()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalled()
// No fireAssign() — drive several polls.
const callsBefore = mockRdKafka.consume.mock.calls.length
for (let i = 0; i < 5; i++) {
if (consumeCallback) {
const cb = consumeCallback
consumeCallback = undefined
cb(null, [])
}
await delay(5)
}
expect(mockRdKafka.consume.mock.calls.length).toBeGreaterThan(callsBefore)
// Loop still running — not crashed.
expect((consumer as any).running).toBe(true)
})
it('Sticky reassign roundtrip: REVOKE [P0] → ASSIGN [P0] back, no stuck or duplicate work', async () => {
// Validates the cooperative-sticky case where the broker takes our partition then
// gives it right back. v1's H2 bug fired here because of a stale deferred .then;
// v2's drain-inside-loop design must handle this cleanly.
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
fireRevoke([{ topic: 'test-topic', partition: 0 }])
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
const unassignCallsBeforeReassign = mockRdKafka.incrementalUnassign.mock.calls.length
fireAssign([{ topic: 'test-topic', partition: 0 }])
await delay(20)
expect(mockRdKafka.incrementalAssign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0 }])
// Critical: no spurious unassign of the just-reassigned partition.
expect(mockRdKafka.incrementalUnassign.mock.calls.length).toBe(unassignCallsBeforeReassign)
})
it('Incremental ASSIGN: a second ASSIGN with new partitions extends the assignment', async () => {
// With cooperative-sticky, a rebalance can deliver an ASSIGN for new partitions
// without first revoking the existing ones. The handler must call
// incrementalAssign() so existing partitions stay assigned.
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch, [{ topic: 'test-topic', partition: 0 }])
// Now a second ASSIGN comes in for partition 1 (cooperative add).
fireAssign([{ topic: 'test-topic', partition: 1 }])
await delay(10)
// incrementalAssign was called twice — once per ASSIGN event — never assign().
expect(mockRdKafka.incrementalAssign).toHaveBeenNthCalledWith(1, [{ topic: 'test-topic', partition: 0 }])
expect(mockRdKafka.incrementalAssign).toHaveBeenNthCalledWith(2, [{ topic: 'test-topic', partition: 1 }])
expect(mockRdKafka.assign).not.toHaveBeenCalled()
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
})
it('Partial REVOKE: only the revoked partition is unassigned, drain awaits in-flight task', async () => {
// Cooperative-sticky drops a subset; we await the in-flight task before
// unassigning, and only call incrementalUnassign for the revoked partition.
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch, [
{ topic: 'test-topic', partition: 0 },
{ topic: 'test-topic', partition: 1 },
])
const p1 = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], p1.promise)
fireRevoke([{ topic: 'test-topic', partition: 1 }])
await delay(5)
// Drain blocks unassign until the in-flight task settles.
expect(mockRdKafka.incrementalUnassign).not.toHaveBeenCalled()
p1.resolve()
await delay(20)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 1 }])
expect((consumer as any).running).toBe(true)
})
it('Disconnect drains in-flight tasks before tearing down', async () => {
;(consumer as any).maxBackgroundTasks = 5
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const slow = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], slow.promise)
const disconnectPromise = consumer.disconnect()
// Force the in-flight consume to release so the loop can observe state=STOPPED.
if (consumeCallback) {
consumeCallback(null, [])
}
// disconnect should be pending until slow resolves.
await delay(20)
expect(mockRdKafka.disconnect).not.toHaveBeenCalled()
slow.resolve()
await disconnectPromise
expect(mockRdKafka.offsetsStore).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0, offset: 2 }])
expect(mockRdKafka.disconnect).toHaveBeenCalled()
// Avoid double-disconnect in afterEach.
consumer = undefined as unknown as KafkaConsumerV2
})
it('eachBatch returning void: offsets still stored', async () => {
const eachBatch = jest.fn(() => Promise.resolve())
await startConsuming(eachBatch)
consumeCallback!(null, [createMessage({ offset: 5, partition: 0 })])
await delay(5)
expect(mockRdKafka.offsetsStore).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0, offset: 6 }])
})
it('eachBatch throwing: propagates the error and kills the loop (matches v1)', async () => {
// v2 deliberately does NOT catch eachBatch errors. The loop crashes, connect()'s
// .catch() re-throws to the orchestrator, the pod restarts, and the batch is
// re-read from the last committed offset (at-least-once preserved).
const error = new Error('boom')
const eachBatch = jest.fn().mockRejectedValue(error)
// Capture the loop promise so we can assert it rejects with our error.
await consumer.connect(eachBatch)
const loopDone = (consumer as any).loopDone as Promise<void>
registeredRebalanceCb!({ code: CODES.ERRORS.ERR__ASSIGN_PARTITIONS } as any, [
{ topic: 'test-topic', partition: 0 },
])
if (consumeCallback) {
const cb = consumeCallback
consumeCallback = undefined
cb(null, [])
}
await delay(5)
consumeCallback!(null, [createMessage({ offset: 7, partition: 0 })])
await expect(loopDone).rejects.toBe(error)
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0, offset: 8 }])
// afterEach calls disconnect() — the loopDone has already settled, so it's a no-op.
})
it('callEachBatchWhenEmpty: empty batches still call eachBatch', async () => {
consumer = (() => {
jest.mocked(require('node-rdkafka').KafkaConsumer)
return new KafkaConsumerV2({
groupId: 'test-group-empty',
topic: 'test-topic',
callEachBatchWhenEmpty: true,
})
})()
mockRdKafka = jest.mocked((consumer as any).rdKafkaConsumer)
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
consumeCallback!(null, [])
await delay(5)
expect(eachBatch).toHaveBeenCalledWith([])
})
it('autoOffsetStore=false: storeOffsets never called automatically', async () => {
consumer = (() => {
return new KafkaConsumerV2({
groupId: 'test-group-manual',
topic: 'test-topic',
autoOffsetStore: false,
})
})()
mockRdKafka = jest.mocked((consumer as any).rdKafkaConsumer)
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
consumeCallback!(null, [createMessage({ offset: 9, partition: 0 })])
await delay(5)
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
// Manual path still works.
consumer.offsetsStore([{ topic: 'test-topic', partition: 0, offset: 10 }])
expect(mockRdKafka.offsetsStore).toHaveBeenCalledWith([{ topic: 'test-topic', partition: 0, offset: 10 }])
})
it('Partition fence: a settle that fires after the partition is given up skips storeOffsets', async () => {
;(consumer as any).maxBackgroundTasks = 5
;(consumer as any).drainTimeoutMs = 30 // force drain timeout so the epoch bump precedes resolve
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
const slow = triggerablePromise()
await dispatchBatch(eachBatch, [createMessage({ offset: 1, partition: 0 })], slow.promise)
fireRevoke()
// Drain times out → loop calls incrementalUnassign → state goes IDLE; the epoch is bumped.
await delay(60)
expect(mockRdKafka.incrementalUnassign).toHaveBeenCalled()
// Resolve the slow task AFTER the epoch bump.
slow.resolve()
await delay(20)
// storeOffsets should NOT be called for this stale task.
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
})
it('backgroundTask rejection: latches fatalError and skips offset store', async () => {
// Loop-exit propagation is covered by the existing "eachBatch throwing" test plus
// the runLoop fatalError guard. Here we pin the new behavior: the rejection is
// latched onto fatalError and offsets for the failed batch are not stored.
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
// Suppress unhandled-rejection warning in case the loop ticks (and throws fatalError)
// during afterEach teardown before disconnect attaches its catch handler.
;((consumer as any).loopDone as Promise<void>).catch(() => {})
const failure = new Error('column "X" does not exist')
const failing = triggerablePromise<void>()
eachBatch.mockImplementationOnce(() => Promise.resolve({ backgroundTask: failing.promise }))
consumeCallback!(null, [createMessage({ offset: 1, partition: 0 })])
await delay(2)
failing.reject(failure)
await delay(10)
expect(captureException).toHaveBeenCalledWith(failure)
expect((consumer as any).fatalError).toBe(failure)
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
})
it('backgroundTask rejection: a later successful batch must NOT advance the bookmark past the failed one', async () => {
// The cross-batch ordering bug: batch N's task fails, batch N+1's succeeds, and
// without the fatalError gate batch N+1 would store its (higher) offset, silently
// committing past the lost events.
;(consumer as any).maxBackgroundTasks = 4
const eachBatch = jest.fn(() => Promise.resolve({}))
const onOffsetsStored = jest.fn()
await startConsuming(eachBatch, undefined, undefined, onOffsetsStored)
;((consumer as any).loopDone as Promise<void>).catch(() => {})
const failure = new Error('column "X" does not exist')
const slowFailure = triggerablePromise<void>()
eachBatch.mockImplementationOnce(() => Promise.resolve({ backgroundTask: slowFailure.promise }))
consumeCallback!(null, [createMessage({ offset: 100, partition: 0 })])
await delay(2)
// Second batch on the same partition with a successful, fast task.
eachBatch.mockImplementationOnce(() => Promise.resolve({ backgroundTask: Promise.resolve() }))
consumeCallback!(null, [createMessage({ offset: 200, partition: 0 })])
await delay(10)
// Resolve the first batch's task with a failure AFTER batch N+1 finished its own work.
slowFailure.reject(failure)
await delay(20)
const partition0Stores = (mockRdKafka.offsetsStore as jest.Mock).mock.calls.filter(([offsets]) =>
offsets.some((o: any) => o.partition === 0)
)
expect(partition0Stores).toEqual([])
expect(onOffsetsStored).not.toHaveBeenCalled()
})
it('fails unfinished background work at the configured timeout without advancing offsets', async () => {
jest.useFakeTimers()
try {
consumer = new KafkaConsumerV2({
groupId: 'test-group',
topic: 'test-topic',
backgroundTaskTimeoutMs: 100,
})
const task = triggerablePromise<void>()
const eachBatch = jest.fn(() => Promise.resolve({ backgroundTask: task.promise }))
const starting = startConsuming(eachBatch)
await jest.advanceTimersByTimeAsync(5)
await starting
const loopFailure = (consumer as unknown as { loopDone: Promise<void> }).loopDone.catch(
(error: unknown) => error
)
consumeCallback!(null, [createMessage({ offset: 1, partition: 0 })])
await jest.advanceTimersByTimeAsync(99)
expect(captureException).not.toHaveBeenCalled()
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
await jest.advanceTimersByTimeAsync(1)
expect(captureException).toHaveBeenCalledWith(new Error('background_task_timeout_after_100ms'))
expect(mockRdKafka.offsetsStore).not.toHaveBeenCalled()
await expect(loopFailure).resolves.toEqual(new Error('background_task_timeout_after_100ms'))
task.resolve()
} finally {
jest.useRealTimers()
}
})
it('Health: not connected → error', () => {
;(mockRdKafka.isConnected as jest.Mock).mockReturnValue(false)
const result = consumer.isHealthy()
expect(result.isError()).toBe(true)
})
it('Health: connected and recently ticked → ok', async () => {
const eachBatch = jest.fn(() => Promise.resolve({}))
await startConsuming(eachBatch)
expect(consumer.isHealthy().isError()).toBe(false)
})
})
afterAll(() => {
// Belt and braces — make sure no tracked promises leak between files.
while (activePromiseResolvers.length > 0) {
activePromiseResolvers[0]()
}
})