Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

26.4 KB
Raw Blame
import { context, propagation, trace } from '@opentelemetry/api'
import { InMemorySpanExporter, NodeTracerProvider, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node'
import { Message } from 'node-rdkafka'

import { BatchBudget } from './batch-budget'
import { BatchingPipeline } from './batching-pipeline'
import { newBatchingPipeline } from './builders/helpers'
import { OkResultWithContext } from './chunk-pipeline.interface'
import { batchBudgetExhaustedCounter } from './metrics'
import { getBudgetsExhausted } from './metrics.test-utils'
import { PipelineResultWithContext } from './pipeline.interface'
import { PipelineResultType, ok } from './results'

type MsgCtx = { message: Message }
function makeMessage(offset: number): Message {
    return {
        topic: 'test-topic',
        partition: 0,
        offset,
        size: 0,
        key: Buffer.from(`key${offset}`),
        value: Buffer.from(`value${offset}`),
        timestamp: Date.now(),
    }
}

function makeBatch(offsets: number[]): OkResultWithContext<any, MsgCtx>[] {
    return offsets.map((offset) => ({
        result: ok({ value: `msg-${offset}` }),
        context: {
            message: makeMessage(offset),
            lastStep: undefined,
            sideEffects: [],
            warnings: [],
        },
    }))
}

describe('BatchingPipeline', () => {
    let beforeBatchStep: jest.Mock
    let afterBatchStep: jest.Mock

    beforeEach(() => {
        beforeBatchStep = jest.fn(({ elements, batchContext }) => Promise.resolve(ok({ elements, batchContext })))
        afterBatchStep = jest.fn((input) =>
            Promise.resolve(ok({ elements: input.elements, batchContext: input.batchContext }))
        )
    })

    function createCollector(options?: { concurrentBatches?: number }) {
        return newBatchingPipeline<any, any, MsgCtx>(
            (builder) => builder.pipe(beforeBatchStep),
            (builder) => builder,
            (builder) => builder.pipe(afterBatchStep),
            { concurrentBatches: Infinity, ...options }
        )
    }

    function createStreamingCollector() {
        return newBatchingPipeline<any, any, MsgCtx>(
            (builder) => builder.pipe(beforeBatchStep),
            (builder) => builder.concurrently((b) => b.pipe((value) => Promise.resolve(ok(value)))),
            (builder) => builder.pipe(afterBatchStep),
            { concurrentBatches: Infinity }
        )
    }

    async function drainAll(
        pipeline: BatchingPipeline<any, any, any, any, any, never>
    ): Promise<{ allResults: PipelineResultWithContext<any, any>[]; allSideEffects: Promise<unknown>[] }> {
        const allResults: PipelineResultWithContext<any, any>[] = []
        const allSideEffects: Promise<unknown>[] = []
        let r = await pipeline.next()
        while (r !== null) {
            allResults.push(...r.elements)
            allSideEffects.push(...(r.sideEffects ?? []))
            r = await pipeline.next()
        }
        return { allResults, allSideEffects }
    }

    it('stamps the fed budget on every element of that batch', async () => {
        const first = BatchBudget.softDeadline(Date.now() + 60_000)
        const second = BatchBudget.softDeadline(Date.now() + 60_000)
        const collector = createCollector()

        await collector.feed(makeBatch([1, 2]), {}, first)
        await collector.feed(makeBatch([3]), {}, second)
        const { allResults } = await drainAll(collector)

        expect(allResults.map((result) => result.context.budget)).toEqual([first, first, second])
    })

    it('leaves a batch unlimited when feed() is given no budget', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([1]), {})
        const { allResults } = await drainAll(collector)

        expect(allResults[0].context.budget).toBe(BatchBudget.unlimited())
    })

    it.each([
        ['before the batch started', -10, 0],
        ['while afterBatch flushed', 500, 1_000],
    ])('counts a budget that expired %s', async (_when, deadlineInMs, flushMs) => {
        jest.useFakeTimers()
        try {
            batchBudgetExhaustedCounter.reset()
            afterBatchStep.mockImplementation((input) => {
                jest.setSystemTime(Date.now() + flushMs)
                return Promise.resolve(ok({ elements: input.elements, batchContext: input.batchContext }))
            })
            const collector = createCollector()

            await collector.feed(makeBatch([1]), {}, BatchBudget.softDeadline(Date.now() + deadlineInMs))
            await drainAll(collector)

            expect(await getBudgetsExhausted()).toBe(1)
        } finally {
            jest.useRealTimers()
        }
    })

    it('returns null when sub-pipeline is empty', async () => {
        const collector = createCollector()
        expect(await collector.next()).toBeNull()
        expect(beforeBatchStep).not.toHaveBeenCalled()
        expect(afterBatchStep).not.toHaveBeenCalled()
    })

    it('assigns sequential batch IDs to beforeBatch', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([1, 2]), {})
        await collector.feed(makeBatch([3]), {})

        expect(beforeBatchStep).toHaveBeenCalledTimes(2)
        expect(beforeBatchStep.mock.calls[0][0].batchContext.batchId).toBe(0)
        expect(beforeBatchStep.mock.calls[0][0].elements).toHaveLength(2)
        expect(beforeBatchStep.mock.calls[1][0].batchContext.batchId).toBe(1)
        expect(beforeBatchStep.mock.calls[1][0].elements).toHaveLength(1)
    })

    it('tags each element with a monotonic messageId in context', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([10, 20, 30]), {})
        const { allResults } = await drainAll(collector)

        expect(allResults).toHaveLength(3)
        expect(allResults[0].context.messageId).toBe(0)
        expect(allResults[1].context.messageId).toBe(1)
        expect(allResults[2].context.messageId).toBe(2)
    })

    it('continues messageId sequence across batches', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([1, 2]), {})
        await collector.feed(makeBatch([3]), {})
        const { allResults } = await drainAll(collector)

        expect(allResults).toHaveLength(3)
        expect(allResults[0].context.messageId).toBe(0)
        expect(allResults[1].context.messageId).toBe(1)
        expect(allResults[2].context.messageId).toBe(2)
    })

    it('returns ordered batch results when a batch completes', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([1, 2, 3]), {})
        const { allResults } = await drainAll(collector)

        expect(afterBatchStep).toHaveBeenCalledTimes(1)
        expect(afterBatchStep.mock.calls[0][0].batchId).toBe(0)
        expect(allResults).toHaveLength(3)
        expect(allResults[0].context.messageId).toBe(0)
        expect(allResults[1].context.messageId).toBe(1)
        expect(allResults[2].context.messageId).toBe(2)
    })

    it('tracks two batches independently', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([1, 2]), {})
        await collector.feed(makeBatch([3]), {})
        const { allResults } = await drainAll(collector)

        expect(afterBatchStep).toHaveBeenCalledTimes(2)
        expect(allResults).toHaveLength(3)
    })

    it('handles single-message batches', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([42]), {})
        const { allResults } = await drainAll(collector)

        expect(beforeBatchStep).toHaveBeenCalledTimes(1)
        expect(afterBatchStep).toHaveBeenCalledTimes(1)
        expect(allResults).toHaveLength(1)
    })

    it('supports feed-drain-feed-drain cycle', async () => {
        const collector = createCollector()

        await collector.feed(makeBatch([1]), {})
        const { allResults: results1 } = await drainAll(collector)
        expect(afterBatchStep).toHaveBeenCalledTimes(1)
        expect(results1).toHaveLength(1)

        await collector.feed(makeBatch([2]), {})
        const { allResults: results2 } = await drainAll(collector)
        expect(afterBatchStep).toHaveBeenCalledTimes(2)
        expect(afterBatchStep.mock.calls[1][0].batchId).toBe(1)
        expect(results2).toHaveLength(1)
    })

    it('awaits async afterBatch before returning from next()', async () => {
        const order: string[] = []

        afterBatchStep.mockImplementation(async (input: any) => {
            order.push('callback-start')
            await new Promise((r) => setTimeout(r, 10))
            order.push('callback-end')
            return ok({ elements: input.elements, batchContext: input.batchContext })
        })

        const collector = createCollector()
        await collector.feed(makeBatch([1]), {})

        order.push('next-called')
        await collector.next()
        order.push('next-resolved')

        expect(order).toEqual(['next-called', 'callback-start', 'callback-end', 'next-resolved'])
    })

    it('runs afterBatch to completion with unchanged results when the budget expires mid-batch', async () => {
        jest.useFakeTimers()
        try {
            const order: string[] = []
            let releaseAfter!: () => void
            const gate = new Promise<void>((resolve) => (releaseAfter = resolve))
            afterBatchStep.mockImplementation(async (input: any) => {
                order.push('after-start')
                await gate
                order.push('after-end')
                return ok({ elements: input.elements, batchContext: input.batchContext })
            })

            const processed: string[] = []
            function processStep(value: { value: string }) {
                processed.push(value.value)
                if (value.value === 'msg-2') {
                    jest.advanceTimersByTime(10_000)
                }
                return Promise.resolve(ok(value))
            }
            const collector = newBatchingPipeline<any, any, MsgCtx>(
                (builder) => builder.pipe(beforeBatchStep),
                (builder) => builder.sequentially((b) => b.pipe(processStep)),
                (builder) => builder.pipe(afterBatchStep),
                { concurrentBatches: Infinity }
            )

            await collector.feed(makeBatch([1, 2, 3]), {}, BatchBudget.softDeadline(Date.now() + 5_000))
            const pendingNext = collector.next().then((batch) => {
                order.push('next-resolved')
                return batch
            })
            for (let i = 0; i < 100 && !order.includes('after-start'); i++) {
                await Promise.resolve()
            }
            expect(order).toContain('after-start')
            expect(order).not.toContain('next-resolved')

            releaseAfter()
            const batch = await pendingNext

            expect(order).toEqual(['after-start', 'after-end', 'next-resolved'])
            // msg-2 started before expiry, so its result stands.
            expect(processed).toEqual(['msg-1', 'msg-2'])
            expect(batch!.elements.map((element) => element.result.type)).toEqual([
                PipelineResultType.OK,
                PipelineResultType.OK,
                PipelineResultType.TIMEOUT,
            ])
            expect(afterBatchStep).toHaveBeenCalledTimes(1)
        } finally {
            jest.useRealTimers()
        }
    })

    describe('with streaming sub-pipeline', () => {
        it('blocks until a batch completes', async () => {
            const collector = createStreamingCollector()

            await collector.feed(makeBatch([1, 2, 3]), {})

            const result = await collector.next()
            expect(result).not.toBeNull()
            expect(result!.elements).toHaveLength(3)
            expect(afterBatchStep).toHaveBeenCalledTimes(1)
        })

        it('fires afterBatch for first batch even when second is still in flight', async () => {
            const collector = createStreamingCollector()

            await collector.feed(makeBatch([1]), {})
            await collector.feed(makeBatch([2, 3]), {})

            const result0 = await collector.next()
            expect(afterBatchStep).toHaveBeenCalledTimes(1)
            expect(afterBatchStep.mock.calls[0][0].batchId).toBe(0)
            expect(result0!.elements).toHaveLength(1)

            const result1 = await collector.next()
            expect(afterBatchStep).toHaveBeenCalledTimes(2)
            expect(afterBatchStep.mock.calls[1][0].batchId).toBe(1)
            expect(result1!.elements).toHaveLength(2)
            expect(result1!.elements[0].context.messageId).toBe(1)
            expect(result1!.elements[1].context.messageId).toBe(2)
        })

        it('returns results from multiple batches completing on the same next() call', async () => {
            const collector = createStreamingCollector()

            await collector.feed(makeBatch([1]), {})
            await collector.feed(makeBatch([2]), {})

            const result0 = await collector.next()
            expect(result0!.elements).toHaveLength(1)

            const result1 = await collector.next()
            expect(result1!.elements).toHaveLength(1)

            expect(afterBatchStep).toHaveBeenCalledTimes(2)
        })
    })

    it('beforeBatch can add extra context to elements', async () => {
        type BatchStore = { batchStore: string }
        const collector = newBatchingPipeline<any, any, MsgCtx, BatchStore>(
            (builder) =>
                builder.pipe(({ elements, batchContext }) =>
                    Promise.resolve(
                        ok({
                            elements: elements.map((el: any) => ({
                                ...el,
                                context: {
                                    ...el.context,
                                    batchStore: `store-for-batch-${batchContext.batchId}`,
                                },
                            })),
                            batchContext: { ...batchContext, batchStore: `store-for-batch-${batchContext.batchId}` },
                        })
                    )
                ),
            (builder) => builder,
            (builder) => builder.pipe((input) => Promise.resolve(ok(input))),
            { concurrentBatches: Infinity }
        )

        await collector.feed(makeBatch([1, 2]), {})
        const { allResults } = await drainAll(collector)

        expect(allResults).toHaveLength(2)
        expect(allResults[0].context).toHaveProperty('batchStore', 'store-for-batch-0')
        expect(allResults[1].context).toHaveProperty('batchStore', 'store-for-batch-0')
        expect(allResults[0].context.messageId).toBe(0)
        expect(allResults[1].context.messageId).toBe(1)
    })

    it('passes batchContext from beforeBatch to afterBatch', async () => {
        type Stores = { personsStore: string; groupStore: string }

        const capturedBefore: Stores[] = []
        const capturedAfter: Stores[] = []
        const collector = newBatchingPipeline<any, any, MsgCtx, Stores>(
            (builder) =>
                builder.pipe(({ elements, batchContext: initBatchContext }) => {
                    const stores: Stores =
                        initBatchContext.batchId === 0
                            ? { personsStore: 'persons-0', groupStore: 'groups-0' }
                            : { personsStore: 'persons-1', groupStore: 'groups-1' }
                    const batchContext = { ...initBatchContext, ...stores }
                    capturedBefore.push(stores)
                    return Promise.resolve(ok({ elements: elements as any, batchContext }))
                }),
            (builder) => builder,
            (builder) =>
                builder.pipe((input) => {
                    capturedAfter.push(input.batchContext)
                    return Promise.resolve(ok(input))
                }),
            { concurrentBatches: Infinity }
        )

        await collector.feed(makeBatch([1]), {})
        await collector.feed(makeBatch([2]), {})
        await drainAll(collector)

        expect(capturedBefore).toEqual([
            { personsStore: 'persons-0', groupStore: 'groups-0' },
            { personsStore: 'persons-1', groupStore: 'groups-1' },
        ])
        expect(capturedAfter).toEqual([
            expect.objectContaining({ personsStore: 'persons-0', groupStore: 'groups-0' }),
            expect.objectContaining({ personsStore: 'persons-1', groupStore: 'groups-1' }),
        ])
    })

    describe('side effects', () => {
        it('collects before pipeline side effects in next() result', async () => {
            const sideEffect = Promise.resolve('before-effect')
            beforeBatchStep.mockImplementation(({ elements, batchContext }: any) =>
                ok({ elements, batchContext }, [sideEffect])
            )

            const collector = createCollector()
            await collector.feed(makeBatch([1]), {})

            const result = await collector.next()
            expect(result).not.toBeNull()
            expect(result!.sideEffects).toContain(sideEffect)
        })

        it('collects after pipeline side effects in next() result', async () => {
            const sideEffect = Promise.resolve('after-effect')
            afterBatchStep.mockImplementation((input: any) =>
                ok({ elements: input.elements, batchContext: input.batchContext }, [sideEffect])
            )

            const collector = createCollector()
            await collector.feed(makeBatch([1]), {})

            const result = await collector.next()
            expect(result).not.toBeNull()
            expect(result!.sideEffects).toContain(sideEffect)
        })

        it('concatenates before and after side effects', async () => {
            const beforeEffect = Promise.resolve('before')
            const afterEffect = Promise.resolve('after')

            beforeBatchStep.mockImplementation(({ elements, batchContext }: any) =>
                ok({ elements, batchContext }, [beforeEffect])
            )
            afterBatchStep.mockImplementation((input: any) =>
                ok({ elements: input.elements, batchContext: input.batchContext }, [afterEffect])
            )

            const collector = createCollector()
            await collector.feed(makeBatch([1]), {})

            const result = await collector.next()
            expect(result).not.toBeNull()
            expect(result!.sideEffects).toEqual([beforeEffect, afterEffect])
        })
    })

    describe('empty and count-changing batches', () => {
        // An empty feed used to register an un-completable batch: next() then
        // threw the "null with N in-flight batches" corruption guard, and the
        // phantom batch permanently occupied a concurrentBatches slot so every
        // later feed() was rejected.
        it('empty feed is a no-op that skips hooks and does not leak a capacity slot', async () => {
            const collector = createCollector({ concurrentBatches: 1 })

            expect(await collector.feed([], {})).toEqual({ ok: true })
            expect(beforeBatchStep).not.toHaveBeenCalled()
            expect(await collector.next()).toBeNull()

            // The slot was not leaked: a subsequent normal batch is accepted and processed.
            expect(await collector.feed(makeBatch([9]), {})).toEqual({ ok: true })
            const { allResults } = await drainAll(collector)
            expect(allResults).toHaveLength(1)
            expect(afterBatchStep).toHaveBeenCalledTimes(1)
        })

        // beforeBatch must preserve the element count; a shrunken batch (worst
        // case zero elements) could never complete and would leak its slot.
        // A count change is a broken framework invariant, so feed() throws.
        it('throws when beforeBatch changes the element count, without registering state', async () => {
            beforeBatchStep.mockImplementationOnce(({ elements, batchContext }: any) =>
                Promise.resolve(ok({ elements: elements.slice(1), batchContext }))
            )
            const collector = createCollector({ concurrentBatches: 1 })

            await expect(collector.feed(makeBatch([1, 2]), {})).rejects.toThrow('changed element count (2 -> 1)')
            expect(await collector.next()).toBeNull()

            // Nothing was registered before the throw: a subsequent normal batch
            // is accepted and processed (real drivers crash on the throw; this
            // documents that the throw itself does not corrupt state).
            expect(await collector.feed(makeBatch([9]), {})).toEqual({ ok: true })
            const { allResults } = await drainAll(collector)
            expect(allResults).toHaveLength(1)
            expect(afterBatchStep).toHaveBeenCalledTimes(1)
        })
    })

    describe('concurrentBatches', () => {
        it('feed() rejects with reason when at limit (default concurrentBatches: 1)', async () => {
            const collector = createCollector({ concurrentBatches: 1 })

            expect(await collector.feed(makeBatch([1]), {})).toEqual({ ok: true })
            expect(await collector.feed(makeBatch([2]), {})).toMatchObject({ ok: false, reason: expect.any(String) })
        })

        it('draining a batch frees a slot', async () => {
            const collector = createCollector({ concurrentBatches: 1 })

            expect((await collector.feed(makeBatch([1]), {})).ok).toBe(true)
            expect((await collector.feed(makeBatch([2]), {})).ok).toBe(false)

            await drainAll(collector)

            expect((await collector.feed(makeBatch([3]), {})).ok).toBe(true)
        })

        it('feed() accepts when under limit and rejects when at limit', async () => {
            const collector = createCollector({ concurrentBatches: 2 })

            expect((await collector.feed(makeBatch([1]), {})).ok).toBe(true)
            expect((await collector.feed(makeBatch([2]), {})).ok).toBe(true)
            expect((await collector.feed(makeBatch([3]), {})).ok).toBe(false)

            await drainAll(collector)

            expect((await collector.feed(makeBatch([4]), {})).ok).toBe(true)
        })
    })

    describe('tracing', () => {
        let exporter: InMemorySpanExporter
        let provider: NodeTracerProvider

        beforeEach(() => {
            exporter = new InMemorySpanExporter()
            provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
            provider.register()
        })

        afterEach(async () => {
            await provider.shutdown()
            trace.disable()
            context.disable()
            propagation.disable()
        })

        it('parents every step span on one span per batch', async () => {
            function markStep(value: any): Promise<ReturnType<typeof ok>> {
                return Promise.resolve(ok(value))
            }
            const collector = newBatchingPipeline<any, any, MsgCtx>(
                (builder) => builder.pipe(beforeBatchStep),
                (builder) => builder.concurrently((b) => b.pipe(markStep)),
                (builder) => builder.pipe(afterBatchStep),
                { concurrentBatches: Infinity }
            )

            const withTeam = (batch: OkResultWithContext<any, MsgCtx>[]): OkResultWithContext<any, MsgCtx>[] =>
                batch.map((element) => ({ ...element, result: ok({ ...element.result.value, teamId: 7 }) }))
            await collector.feed(withTeam(makeBatch([1, 2])), {})
            await collector.feed(withTeam(makeBatch([3])), {})
            await drainAll(collector)

            const spans = exporter.getFinishedSpans()
            const batchSpans = spans.filter((span) => span.name === 'batchingPipeline.batch')
            expect(batchSpans).toHaveLength(2)
            expect(batchSpans.map((span) => span.parentSpanContext)).toEqual([undefined, undefined])
            expect(batchSpans.map((span) => span.attributes.batch_size)).toEqual([2, 1])

            const parentOf = (span: { parentSpanContext?: { spanId: string } }): string | undefined =>
                span.parentSpanContext?.spanId
            const batchSpanIds = batchSpans.map((span) => span.spanContext().spanId)
            const stepSpans = spans.filter((span) => span.name === 'markStep')
            expect(stepSpans.map(parentOf)).toEqual([batchSpanIds[0], batchSpanIds[0], batchSpanIds[1]])
            expect(stepSpans.map((span) => span.attributes.team_id)).toEqual([7, 7, 7])

            const hookSpans = spans.filter((span) => span.name !== 'batchingPipeline.batch' && span.name !== 'markStep')
            expect(hookSpans).toHaveLength(4)
            expect(hookSpans.map(parentOf).sort()).toEqual([...batchSpanIds, ...batchSpanIds].sort())
        })

        it('gives every batch in a mixed chunk its own chunk span', async () => {
            function tagChunk(values: any[]): Promise<ReturnType<typeof ok>[]> {
                return Promise.resolve(values.map((value) => ok(value)))
            }
            const collector = newBatchingPipeline<any, any, MsgCtx>(
                (builder) => builder.pipe(beforeBatchStep),
                (builder) => builder.pipeChunk(tagChunk),
                (builder) => builder.pipe(afterBatchStep),
                { concurrentBatches: Infinity }
            )

            await collector.feed(makeBatch([1, 2]), {})
            await collector.feed(makeBatch([3]), {})
            await drainAll(collector)

            const spans = exporter.getFinishedSpans()
            const batchSpanIds = spans
                .filter((span) => span.name === 'batchingPipeline.batch')
                .map((span) => span.spanContext().spanId)
            const chunkSpans = spans.filter((span) => span.name === 'tagChunk')
            expect(chunkSpans.map((span) => span.attributes.chunk_size)).toEqual([3, 3])
            expect(chunkSpans.map((span) => span.parentSpanContext?.spanId).sort()).toEqual([...batchSpanIds].sort())
        })

        it('ends the batch span when the pull fails', async () => {
            function failStep(): Promise<never> {
                return Promise.reject(new Error('boom'))
            }
            const collector = newBatchingPipeline<any, any, MsgCtx>(
                (builder) => builder.pipe(beforeBatchStep),
                (builder) => builder.concurrently((b) => b.pipe(failStep)),
                (builder) => builder.pipe(afterBatchStep),
                { concurrentBatches: Infinity }
            )

            await collector.feed(makeBatch([1]), {})
            await expect(drainAll(collector)).rejects.toThrow('boom')

            expect(exporter.getFinishedSpans().map((span) => span.name)).toContain('batchingPipeline.batch')
        })
    })
})