Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

25.9 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { logger } from '~/common/utils/logger'
import { createMockPipeline } from '~/tests/helpers/mock-pipeline'

import { BaseChunkPipeline } from './base-chunk-pipeline'
import { BatchBudget } from './batch-budget'
import {
    aggregateKafkaDebugContexts,
    createBatch,
    createContext,
    createKafkaDebugContext,
    createNewChunkPipeline,
    createOkContext,
} from './helpers'
import { batchBudgetCheckpointCounter } from './metrics'
import { getBudgetCheckpoints } from './metrics.test-utils'
import { dlq, drop, isTimeoutResult, ok } from './results'

jest.mock('~/common/utils/logger', () => ({
    logger: { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn() },
}))

function createTestMessage(overrides: Partial<Message> = {}): Message {
    return {
        value: Buffer.from('test'),
        topic: 'test',
        partition: 0,
        offset: 1,
        key: Buffer.from('key1'),
        size: 4,
        timestamp: Date.now(),
        headers: [],
        ...overrides,
    }
}

describe('BaseChunkPipeline', () => {
    beforeEach(() => {
        batchBudgetCheckpointCounter.reset()
    })

    describe('basic functionality', () => {
        it('times out the exhausted elements of a mixed-budget chunk and runs the step on the rest', async () => {
            const exhausted = BatchBudget.softDeadline(Date.now() - 1)
            const healthy = BatchBudget.softDeadline(Date.now() + 5000)
            let seen: { id: number }[] = []

            const rootPipeline = createNewChunkPipeline<{ id: number }, { budget: BatchBudget }>().build()
            const pipeline = new BaseChunkPipeline(function enrichChunk(items: { id: number }[]) {
                seen = items
                return Promise.resolve(items.map((item) => ok({ id: item.id * 10 })))
            }, rootPipeline)

            pipeline.feed([
                createOkContext({ id: 1 }, { budget: exhausted }),
                createOkContext({ id: 2 }, { budget: healthy }),
            ])
            const results = await pipeline.next()

            expect(seen).toEqual([{ id: 2 }])
            expect(isTimeoutResult(results![0].result)).toBe(true)
            expect((results![0].result as { reason: string }).reason).toBe('budget exceeded before enrichChunk')
            expect(results![1].result).toEqual(ok({ id: 20 }))
            expect(await getBudgetCheckpoints('chunk', 'enrichChunk')).toBe(1)
        })

        it('should process batch through pipeline', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('test1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('test2'), offset: 2 }),
            ]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map((item: any) => ok({ processed: item.message.value?.toString() })))
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toEqual([
                createContext(ok({ processed: 'test1' }), {
                    message: messages[0],
                    debugContext: createKafkaDebugContext(messages[0]),
                    lastStep: 'anonymousChunkStep',
                }),
                createContext(ok({ processed: 'test2' }), {
                    message: messages[1],
                    debugContext: createKafkaDebugContext(messages[1]),
                    lastStep: 'anonymousChunkStep',
                }),
            ])
        })

        it('should handle empty batch', async () => {
            const rootPipeline = createNewChunkPipeline().build()
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map((item: any) => ok(item)))
            }, rootPipeline)

            pipeline.feed([])
            const results = await pipeline.next()

            expect(results).toEqual(null)
        })
    })

    describe('chunk operations', () => {
        it('should execute chunk step on all successful values', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('2'), offset: 2 }),
                createTestMessage({ value: Buffer.from('3'), offset: 3 }),
            ]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(
                    items.map((item: any) => ok({ count: parseInt(item.message.value?.toString() || '0') * 2 }))
                )
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toEqual([
                createContext(ok({ count: 2 }), {
                    message: messages[0],
                    debugContext: createKafkaDebugContext(messages[0]),
                    lastStep: 'anonymousChunkStep',
                }),
                createContext(ok({ count: 4 }), {
                    message: messages[1],
                    debugContext: createKafkaDebugContext(messages[1]),
                    lastStep: 'anonymousChunkStep',
                }),
                createContext(ok({ count: 6 }), {
                    message: messages[2],
                    debugContext: createKafkaDebugContext(messages[2]),
                    lastStep: 'anonymousChunkStep',
                }),
            ])
        })

        it('should preserve non-success results and only process successful ones', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('drop'), offset: 2 }),
                createTestMessage({ value: Buffer.from('3'), offset: 3 }),
                createTestMessage({ value: Buffer.from('dlq'), offset: 4 }),
            ]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()
            const firstPipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(
                    items.map((item: any) => {
                        const value = item.message.value?.toString() || ''
                        if (value === 'drop') {
                            return drop('dropped item')
                        }
                        if (value === 'dlq') {
                            return dlq('dlq item', new Error('test error'))
                        }
                        return ok({ count: parseInt(value) })
                    })
                )
            }, rootPipeline)

            const secondPipeline = new BaseChunkPipeline((items: any[]) => {
                expect(items).toEqual([{ count: 1 }, { count: 3 }])
                return Promise.resolve(items.map((item: any) => ok({ count: item.count * 2 })))
            }, firstPipeline)

            secondPipeline.feed(batch)
            const results = await secondPipeline.next()

            expect(results).toEqual([
                createContext(ok({ count: 2 }), {
                    message: messages[0],
                    debugContext: createKafkaDebugContext(messages[0]),
                    lastStep: 'anonymousChunkStep',
                }),
                createContext(drop('dropped item'), {
                    message: messages[1],
                    debugContext: createKafkaDebugContext(messages[1]),
                    lastStep: 'anonymousChunkStep',
                }),
                createContext(ok({ count: 6 }), {
                    message: messages[2],
                    debugContext: createKafkaDebugContext(messages[2]),
                    lastStep: 'anonymousChunkStep',
                }),
                createContext(dlq('dlq item', new Error('test error')), {
                    message: messages[3],
                    debugContext: createKafkaDebugContext(messages[3]),
                    lastStep: 'anonymousChunkStep',
                }),
            ])
        })
    })

    describe('error handling', () => {
        it('should propagate errors from chunk operations and log aggregated debug contexts', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('2'), offset: 2 }),
                createTestMessage({ value: Buffer.from('3'), offset: 5 }),
            ]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()
            const pipeline = new BaseChunkPipeline(
                () => {
                    return Promise.reject(new Error('Chunk step failed'))
                },
                rootPipeline,
                { aggregateDebugContexts: aggregateKafkaDebugContexts }
            )

            pipeline.feed(batch)
            await expect(pipeline.next()).rejects.toThrow('Chunk step failed')
            expect(logger.error).toHaveBeenCalledWith(
                '🔥',
                expect.stringContaining('threw'),
                expect.objectContaining({
                    error: 'Chunk step failed',
                    debugContext: { count: 3, offsets: ['test[0]:1-2,5'] },
                })
            )
        })
    })

    describe('step name tracking', () => {
        it('should include step name in context for successful results', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('test1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('test2'), offset: 2 }),
            ]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()

            function testChunkStep(items: any[]) {
                return Promise.resolve(items.map((item: any) => ok({ processed: item.message.value?.toString() })))
            }

            const pipeline = new BaseChunkPipeline(testChunkStep, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toEqual([
                createContext(ok({ processed: 'test1' }), {
                    message: messages[0],
                    debugContext: createKafkaDebugContext(messages[0]),
                    lastStep: 'testChunkStep',
                }),
                createContext(ok({ processed: 'test2' }), {
                    message: messages[1],
                    debugContext: createKafkaDebugContext(messages[1]),
                    lastStep: 'testChunkStep',
                }),
            ])
        })

        it('should use anonymousChunkStep when step has no name', async () => {
            const messages: Message[] = [createTestMessage({ value: Buffer.from('test1'), offset: 1 })]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()

            const anonymousStep = (items: any[]) => {
                return Promise.resolve(items.map((item: any) => ok({ processed: item.message.value?.toString() })))
            }

            const pipeline = new BaseChunkPipeline(anonymousStep, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toEqual([
                createContext(ok({ processed: 'test1' }), {
                    message: messages[0],
                    debugContext: createKafkaDebugContext(messages[0]),
                    lastStep: 'anonymousStep',
                }),
            ])
        })

        it('should not update lastStep for failed results', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('test1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('drop'), offset: 2 }),
            ]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()

            function testChunkStep(items: any[]) {
                return Promise.resolve(
                    items.map((item: any) => {
                        const value = item.message.value?.toString() || ''
                        if (value === 'drop') {
                            return drop('dropped item')
                        }
                        return ok({ processed: value })
                    })
                )
            }

            const pipeline = new BaseChunkPipeline(testChunkStep, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toEqual([
                createContext(ok({ processed: 'test1' }), {
                    message: messages[0],
                    debugContext: createKafkaDebugContext(messages[0]),
                    lastStep: 'testChunkStep',
                }),
                createContext(drop('dropped item'), {
                    message: messages[1],
                    debugContext: createKafkaDebugContext(messages[1]),
                    lastStep: 'testChunkStep',
                }),
            ])
        })
    })

    describe('side effects accumulation', () => {
        it('should accumulate side effects from previous context and current step result', async () => {
            const messages: Message[] = [createTestMessage({ value: Buffer.from('test'), offset: 1 })]

            const initialSideEffect1 = Promise.resolve('initial-side-effect-1')
            const initialSideEffect2 = Promise.resolve('initial-side-effect-2')
            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [initialSideEffect1, initialSideEffect2],
                    }
                ),
            ]

            const rootPipeline = createNewChunkPipeline().build()

            const stepSideEffect1 = Promise.resolve('step-side-effect-1')
            const stepSideEffect2 = Promise.resolve('step-side-effect-2')
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map(() => ok({ processed: 'result' }, [stepSideEffect1, stepSideEffect2])))
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(1)
            expect(results![0].context.sideEffects).toEqual([
                initialSideEffect1,
                initialSideEffect2,
                stepSideEffect1,
                stepSideEffect2,
            ])
        })

        it('should preserve context side effects when step returns no side effects', async () => {
            const messages: Message[] = [createTestMessage({ value: Buffer.from('test'), offset: 1 })]

            const existingSideEffect = Promise.resolve('existing-side-effect')
            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [existingSideEffect],
                    }
                ),
            ]

            const rootPipeline = createNewChunkPipeline().build()
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map(() => ok({ processed: 'result' })))
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(1)
            expect(results![0].context.sideEffects).toEqual([existingSideEffect])
        })

        it('should add step side effects when context has no existing side effects', async () => {
            const messages: Message[] = [createTestMessage({ value: Buffer.from('test'), offset: 1 })]

            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [],
                    }
                ),
            ]

            const rootPipeline = createNewChunkPipeline().build()

            const stepSideEffect = Promise.resolve('step-side-effect')
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map(() => ok({ processed: 'result' }, [stepSideEffect])))
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(1)
            expect(results![0].context.sideEffects).toEqual([stepSideEffect])
        })

        it('should handle multiple items with different side effect patterns', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('item1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('item2'), offset: 2 }),
                createTestMessage({ value: Buffer.from('item3'), offset: 3 }),
            ]

            const sideEffect1 = Promise.resolve('context-1')
            const sideEffect2 = Promise.resolve('context-2')
            const sideEffect3 = Promise.resolve('context-3')

            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [sideEffect1],
                    }
                ),
                createOkContext(
                    { message: messages[1] },
                    {
                        message: messages[1],
                        sideEffects: [sideEffect2, sideEffect3],
                    }
                ),
                createOkContext(
                    { message: messages[2] },
                    {
                        message: messages[2],
                        sideEffects: [],
                    }
                ),
            ]

            const rootPipeline = createNewChunkPipeline().build()

            const step1SideEffect = Promise.resolve('step-1')
            const step3aSideEffect = Promise.resolve('step-3a')
            const step3bSideEffect = Promise.resolve('step-3b')
            const pipeline = new BaseChunkPipeline((_: any[]) => {
                return Promise.resolve([
                    ok({ processed: 'result1' }, [step1SideEffect]),
                    ok({ processed: 'result2' }), // No step side effects
                    ok({ processed: 'result3' }, [step3aSideEffect, step3bSideEffect]),
                ])
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(3)

            // First item: context + step side effects
            expect(results![0].context.sideEffects).toEqual([sideEffect1, step1SideEffect])

            // Second item: only context side effects (step has none)
            expect(results![1].context.sideEffects).toEqual([sideEffect2, sideEffect3])

            // Third item: only step side effects (context has none)
            expect(results![2].context.sideEffects).toEqual([step3aSideEffect, step3bSideEffect])
        })
    })

    describe('warning accumulation', () => {
        it('should accumulate warnings from step results', async () => {
            const messages: Message[] = [createTestMessage({ value: Buffer.from('test1'), offset: 1 })]

            const batch = createBatch(messages.map((message) => ({ message })))
            const rootPipeline = createNewChunkPipeline().build()

            const stepWarning = { type: 'merge_race_condition' as const, details: { message: 'step warning' } }
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map(() => ok({ processed: 'result' }, [], [stepWarning])))
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(1)
            expect(results![0].context.warnings).toEqual([stepWarning])
        })

        it('should merge context warnings with step warnings', async () => {
            const messages: Message[] = [createTestMessage({ value: Buffer.from('test1'), offset: 1 })]

            const contextWarning = { type: 'client_ingestion_warning' as const, details: { message: 'from context' } }
            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [],
                        warnings: [contextWarning],
                    }
                ),
            ]

            const rootPipeline = createNewChunkPipeline().build()

            const stepWarning = { type: 'schema_validation_failed' as const, details: { message: 'from step' } }
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                return Promise.resolve(items.map(() => ok({ processed: 'result' }, [], [stepWarning])))
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(1)
            expect(results![0].context.warnings).toEqual([contextWarning, stepWarning])
        })

        it('should handle multiple warnings from multiple items', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('item1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('item2'), offset: 2 }),
                createTestMessage({ value: Buffer.from('item3'), offset: 3 }),
            ]

            const contextWarning1 = { type: 'ignored_invalid_timestamp' as const, details: { idx: 1 } }
            const contextWarning2 = { type: 'invalid_heatmap_data' as const, details: { idx: 2 } }

            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [],
                        warnings: [contextWarning1],
                    }
                ),
                createOkContext(
                    { message: messages[1] },
                    {
                        message: messages[1],
                        sideEffects: [],
                        warnings: [contextWarning2],
                    }
                ),
                createOkContext(
                    { message: messages[2] },
                    {
                        message: messages[2],
                        sideEffects: [],
                        warnings: [],
                    }
                ),
            ]

            const rootPipeline = createNewChunkPipeline().build()

            const stepWarning1 = { type: 'message_size_too_large' as const, details: { result: 1 } }
            const stepWarning3a = { type: 'group_key_too_long' as const, details: { result: 3 } }
            const stepWarning3b = { type: 'event_dropped_too_old' as const, details: { result: 3 } }
            const pipeline = new BaseChunkPipeline((_: any[]) => {
                return Promise.resolve([
                    ok({ processed: 'result1' }, [], [stepWarning1]),
                    ok({ processed: 'result2' }), // No step warnings
                    ok({ processed: 'result3' }, [], [stepWarning3a, stepWarning3b]),
                ])
            }, rootPipeline)

            pipeline.feed(batch)
            const results = await pipeline.next()

            expect(results).toHaveLength(3)

            // First item: context + step warnings
            expect(results![0].context.warnings).toEqual([contextWarning1, stepWarning1])

            // Second item: only context warnings (step has none)
            expect(results![1].context.warnings).toEqual([contextWarning2])

            // Third item: only step warnings (context has none)
            expect(results![2].context.warnings).toEqual([stepWarning3a, stepWarning3b])
        })

        it('should preserve warnings for non-success results', async () => {
            const messages: Message[] = [
                createTestMessage({ value: Buffer.from('1'), offset: 1 }),
                createTestMessage({ value: Buffer.from('drop'), offset: 2 }),
            ]

            const contextWarning = { type: 'client_ingestion_warning' as const, details: { message: 'existing' } }
            const batch = [
                createOkContext(
                    { message: messages[0] },
                    {
                        message: messages[0],
                        sideEffects: [],
                        warnings: [contextWarning],
                    }
                ),
                createContext(drop('drop_reason'), {
                    message: messages[1],
                    sideEffects: [],
                    warnings: [contextWarning],
                }),
            ]

            const mockPrevious = createMockPipeline(batch)
            const pipeline = new BaseChunkPipeline((items: any[]) => {
                // Only process OK results
                return Promise.resolve(items.map(() => ok({ processed: 'result' })))
            }, mockPrevious)

            const results = await pipeline.next()

            expect(results).toHaveLength(2)

            // OK result should have warnings accumulated
            expect(results![0].context.warnings).toEqual([contextWarning])

            // Drop result should preserve context warnings
            expect(results![1].context.warnings).toEqual([contextWarning])
        })
    })
})