Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

30.3 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { DLQ_OUTPUT, INGESTION_WARNINGS_OUTPUT, OVERFLOW_OUTPUT } from '~/common/outputs'
import { OverflowOutput } from '~/common/outputs'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { ingestionPipelineResultCounter } from '~/ingestion/common/metrics'
import {
    logDroppedMessage,
    produceMessageToDLQ,
    redirectMessageToOutput,
} from '~/ingestion/framework/result-handling-helpers'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
import { createMockPipeline } from '~/tests/helpers/mock-pipeline'

import { createContext } from './helpers'
import { PipelineConfig, ResultHandlingPipeline } from './result-handling-pipeline'
import { dlq, drop, ok, redirect, timeout } from './results'

// Mock the pipeline helpers
jest.mock('~/ingestion/framework/result-handling-helpers', () => ({
    logDroppedMessage: jest.fn(),
    redirectMessageToOutput: jest.fn(),
    produceMessageToDLQ: jest.fn(),
}))

// Mock the metrics
jest.mock('~/ingestion/common/metrics', () => ({
    ingestionPipelineResultCounter: {
        labels: jest.fn().mockReturnValue({
            inc: jest.fn(),
        }),
    },
}))

const mockLogDroppedMessage = logDroppedMessage as jest.MockedFunction<typeof logDroppedMessage>
const mockRedirectMessageToOutput = redirectMessageToOutput as jest.MockedFunction<typeof redirectMessageToOutput>
const mockProduceMessageToDLQ = produceMessageToDLQ as jest.MockedFunction<typeof produceMessageToDLQ>
const mockIngestionPipelineResultCounter = ingestionPipelineResultCounter as jest.Mocked<
    typeof ingestionPipelineResultCounter
>

describe('ResultHandlingPipeline', () => {
    let mockPromiseScheduler: PromiseScheduler
    let config: PipelineConfig<OverflowOutput>

    beforeEach(() => {
        jest.clearAllMocks()

        mockPromiseScheduler = {
            schedule: jest.fn(),
        } as unknown as PromiseScheduler

        config = {
            outputs: createMockIngestionOutputs<
                typeof DLQ_OUTPUT | typeof OVERFLOW_OUTPUT | typeof INGESTION_WARNINGS_OUTPUT
            >(),
            promiseScheduler: mockPromiseScheduler,
        }
    })

    describe('basic functionality', () => {
        it('should process successful results and return chunk pipeline results', async () => {
            const messages: Message[] = [
                { value: Buffer.from('test1'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('test2'), topic: 'test', partition: 0, offset: 2 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [
                createContext(ok({ processed: 'test1', message: messages[0] }), { message: messages[0] }),
                createContext(ok({ processed: 'test2', message: messages[1] }), { message: messages[1] }),
            ]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            expect(results).not.toBeNull()
            expect(results).toEqual([
                createContext(ok({ processed: 'test1', message: messages[0] }), { message: messages[0] }),
                createContext(ok({ processed: 'test2', message: messages[1] }), { message: messages[1] }),
            ])
        })

        it('should handle empty chunk', async () => {
            const mockPipeline = createMockPipeline([])
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            expect(results).toBeNull()
        })
    })

    describe('result handling', () => {
        it('should handle dropped results and log them', async () => {
            const messages: Message[] = [
                { value: Buffer.from('test1'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('drop'), topic: 'test', partition: 0, offset: 2 } as Message,
                { value: Buffer.from('test3'), topic: 'test', partition: 0, offset: 3 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [
                createContext(ok({ processed: 'test1' }), { message: messages[0] }),
                createContext(drop('test drop reason'), { message: messages[1] }),
                createContext(ok({ processed: 'test3' }), { message: messages[2] }),
            ]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            expect(results).not.toBeNull()
            expect(results).toEqual([
                createContext(ok({ processed: 'test1' }), { message: messages[0] }),
                createContext(drop('test drop reason'), { message: messages[1] }),
                createContext(ok({ processed: 'test3' }), { message: messages[2] }),
            ])
            expect(mockLogDroppedMessage).toHaveBeenCalledWith(messages[1], 'test drop reason', 'unknown')
        })

        it('should handle redirected results and add side effects', async () => {
            mockRedirectMessageToOutput.mockResolvedValue(undefined)

            const messages: Message[] = [
                { value: Buffer.from('test1'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('redirect'), topic: 'test', partition: 0, offset: 2 } as Message,
                { value: Buffer.from('test3'), topic: 'test', partition: 0, offset: 3 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [
                createContext(ok({ processed: 'test1' }), { message: messages[0] }),
                createContext(redirect('test redirect', OVERFLOW_OUTPUT, true, false), { message: messages[1] }),
                createContext(ok({ processed: 'test3' }), { message: messages[2] }),
            ]

            const mockPipeline = createMockPipeline<any, { message: Message }, OverflowOutput>(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return all results including redirect result with side effects
            expect(results).not.toBeNull()
            expect(results).toHaveLength(3)
            expect(results![0]).toEqual(createContext(ok({ processed: 'test1' }), { message: messages[0] }))
            expect(results![1].result).toEqual(redirect('test redirect', OVERFLOW_OUTPUT, true, false))
            expect(results![1].context.sideEffects).toHaveLength(1)
            expect(results![2]).toEqual(createContext(ok({ processed: 'test3' }), { message: messages[2] }))

            // Extract and await side effects
            const sideEffects = results![1].context.sideEffects
            await Promise.all(sideEffects)

            // Now verify the mock was called
            expect(mockRedirectMessageToOutput).toHaveBeenCalledWith(
                config.outputs,
                OVERFLOW_OUTPUT,
                mockPromiseScheduler,
                messages[1],
                'unknown',
                true,
                false
            )
        })

        it('should redirect messages with event and uuid headers', async () => {
            mockRedirectMessageToOutput.mockResolvedValue(undefined)

            const messagesWithHeaders: Message[] = [
                {
                    value: Buffer.from('redirect'),
                    topic: 'test',
                    partition: 0,
                    offset: 1,
                    size: 8,
                    timestamp: 1234567891,
                    headers: [
                        { distinct_id: Buffer.from('user-456') },
                        { token: Buffer.from('redirect-token') },
                        { event: Buffer.from('$identify') },
                        { uuid: Buffer.from('redirect-uuid-456') },
                    ],
                },
            ]

            const chunkResults = [
                createContext(redirect('test redirect', OVERFLOW_OUTPUT, false, true), {
                    message: messagesWithHeaders[0],
                }),
            ]

            const mockPipeline = createMockPipeline<any, { message: Message }, OverflowOutput>(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return the redirect result with side effects
            expect(results).not.toBeNull()
            expect(results).toHaveLength(1)
            expect(results![0].result).toEqual(redirect('test redirect', OVERFLOW_OUTPUT, false, true))
            expect(results![0].context.sideEffects).toHaveLength(1)

            // Extract and await side effects
            const sideEffects = results![0].context.sideEffects
            await Promise.all(sideEffects)

            // Now verify the mock was called
            expect(mockRedirectMessageToOutput).toHaveBeenCalledWith(
                config.outputs,
                OVERFLOW_OUTPUT,
                mockPromiseScheduler,
                messagesWithHeaders[0],
                'unknown',
                false,
                true
            )

            // Verify the message passed to redirect has the correct headers
            const calledMessage = (mockRedirectMessageToOutput as jest.Mock).mock.calls[0][3]
            expect(calledMessage.headers).toEqual([
                { distinct_id: Buffer.from('user-456') },
                { token: Buffer.from('redirect-token') },
                { event: Buffer.from('$identify') },
                { uuid: Buffer.from('redirect-uuid-456') },
            ])
        })

        it('should handle dlq results and add side effects', async () => {
            mockProduceMessageToDLQ.mockResolvedValue(undefined)

            const messages: Message[] = [
                { value: Buffer.from('test1'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('dlq'), topic: 'test', partition: 0, offset: 2 } as Message,
                { value: Buffer.from('test3'), topic: 'test', partition: 0, offset: 3 } as Message,
            ]

            const testError = new Error('test error')
            // Create chunk results directly
            const chunkResults = [
                createContext(ok({ processed: 'test1' }), { message: messages[0] }),
                createContext(dlq('test dlq reason', testError), { message: messages[1] }),
                createContext(ok({ processed: 'test3' }), { message: messages[2] }),
            ]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return all results including dlq result with side effects
            expect(results).not.toBeNull()
            expect(results).toHaveLength(3)
            expect(results![0]).toEqual(createContext(ok({ processed: 'test1' }), { message: messages[0] }))
            expect(results![1].result).toEqual(dlq('test dlq reason', testError))
            expect(results![1].context.sideEffects).toHaveLength(1)
            expect(results![2]).toEqual(createContext(ok({ processed: 'test3' }), { message: messages[2] }))

            // Extract and await side effects
            const sideEffects = results![1].context.sideEffects
            await Promise.all(sideEffects)

            // Now verify the mock was called
            expect(mockProduceMessageToDLQ).toHaveBeenCalledWith(config.outputs, messages[1], testError, 'unknown')
        })

        it('passes rethrowOnFailure to the DLQ produce when rejectOnDlqFailure is set', async () => {
            const dlqError = new Error('broker down')
            mockProduceMessageToDLQ.mockRejectedValueOnce(dlqError)
            const message = { value: Buffer.from('dlq'), topic: 'test', partition: 0, offset: 1 } as Message
            const testError = new Error('test error')

            const resultPipeline = new ResultHandlingPipeline(
                createMockPipeline([createContext(dlq('test dlq reason', testError), { message })]),
                { ...config, rejectOnDlqFailure: true }
            )
            const results = await resultPipeline.next()

            await expect(Promise.all(results![0].context.sideEffects)).rejects.toBe(dlqError)
            expect(mockProduceMessageToDLQ).toHaveBeenCalledWith(config.outputs, message, testError, 'unknown', {
                rethrowOnFailure: true,
            })
        })

        it('should send DLQ messages with event and uuid headers', async () => {
            mockProduceMessageToDLQ.mockResolvedValue(undefined)

            const messagesWithHeaders: Message[] = [
                {
                    value: Buffer.from('dlq'),
                    topic: 'test',
                    partition: 0,
                    offset: 1,
                    size: 3,
                    timestamp: 1234567890,
                    headers: [
                        { distinct_id: Buffer.from('user-123') },
                        { token: Buffer.from('test-token') },
                        { event: Buffer.from('$pageview') },
                        { uuid: Buffer.from('event-uuid-123') },
                    ],
                },
            ]

            const testError = new Error('test error')
            const chunkResults = [createContext(dlq('test dlq reason', testError), { message: messagesWithHeaders[0] })]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return the dlq result with side effects
            expect(results).not.toBeNull()
            expect(results).toHaveLength(1)
            expect(results![0].result).toEqual(dlq('test dlq reason', testError))
            expect(results![0].context.sideEffects).toHaveLength(1)

            // Extract and await side effects
            const sideEffects = results![0].context.sideEffects
            await Promise.all(sideEffects)

            // Now verify the mock was called
            expect(mockProduceMessageToDLQ).toHaveBeenCalledWith(
                config.outputs,
                messagesWithHeaders[0],
                testError,
                'unknown'
            )

            // Verify the message passed to DLQ has the correct headers
            const calledMessage = (mockProduceMessageToDLQ as jest.Mock).mock.calls[0][1]
            expect(calledMessage.headers).toEqual([
                { distinct_id: Buffer.from('user-123') },
                { token: Buffer.from('test-token') },
                { event: Buffer.from('$pageview') },
                { uuid: Buffer.from('event-uuid-123') },
            ])
        })

        it('should handle dlq result without error and create default error', async () => {
            mockProduceMessageToDLQ.mockResolvedValue(undefined)

            const messages: Message[] = [
                { value: Buffer.from('dlq'), topic: 'test', partition: 0, offset: 1 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [createContext(dlq('test dlq reason'), { message: messages[0] })]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return the dlq result with side effects
            expect(results).not.toBeNull()
            expect(results).toHaveLength(1)
            expect(results![0].result).toEqual(dlq('test dlq reason'))
            expect(results![0].context.sideEffects).toHaveLength(1)

            // Extract and await side effects
            const sideEffects = results![0].context.sideEffects
            await Promise.all(sideEffects)

            // Now verify the mock was called
            expect(mockProduceMessageToDLQ).toHaveBeenCalledWith(
                config.outputs,
                messages[0],
                expect.any(Error),
                'unknown'
            )

            const errorArg = (mockProduceMessageToDLQ as jest.Mock).mock.calls[0][2]
            expect(errorArg.message).toBe('test dlq reason')
        })

        it('should handle mixed results correctly', async () => {
            mockRedirectMessageToOutput.mockResolvedValue(undefined)
            mockProduceMessageToDLQ.mockResolvedValue(undefined)

            const messages: Message[] = [
                { value: Buffer.from('success1'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('drop'), topic: 'test', partition: 0, offset: 2 } as Message,
                { value: Buffer.from('success2'), topic: 'test', partition: 0, offset: 3 } as Message,
                { value: Buffer.from('redirect'), topic: 'test', partition: 0, offset: 4 } as Message,
                { value: Buffer.from('dlq'), topic: 'test', partition: 0, offset: 5 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [
                createContext(ok({ processed: 'success1' }), { message: messages[0] }),
                createContext(drop('dropped item'), { message: messages[1] }),
                createContext(ok({ processed: 'success2' }), { message: messages[2] }),
                createContext(redirect('redirected item', OVERFLOW_OUTPUT), { message: messages[3] }),
                createContext(dlq('dlq item', new Error('processing error')), { message: messages[4] }),
            ]

            const mockPipeline = createMockPipeline<any, { message: Message }, OverflowOutput>(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return all results including non-success ones
            expect(results).not.toBeNull()
            expect(results).toHaveLength(5)
            expect(results![0]).toEqual(createContext(ok({ processed: 'success1' }), { message: messages[0] }))
            expect(results![1]).toEqual(createContext(drop('dropped item'), { message: messages[1] }))
            expect(results![2]).toEqual(createContext(ok({ processed: 'success2' }), { message: messages[2] }))
            expect(results![3].result).toEqual(redirect('redirected item', OVERFLOW_OUTPUT))
            expect(results![3].context.sideEffects).toHaveLength(1)
            expect(results![4].result).toEqual(dlq('dlq item', new Error('processing error')))
            expect(results![4].context.sideEffects).toHaveLength(1)

            // Extract and await side effects from redirect and dlq results
            const redirectSideEffects = results![3].context.sideEffects
            const dlqSideEffects = results![4].context.sideEffects
            await Promise.all([...redirectSideEffects, ...dlqSideEffects])

            // Verify all non-success results were handled
            expect(mockLogDroppedMessage).toHaveBeenCalledWith(messages[1], 'dropped item', 'unknown')
            expect(mockRedirectMessageToOutput).toHaveBeenCalledWith(
                config.outputs,
                OVERFLOW_OUTPUT,
                mockPromiseScheduler,
                messages[3],
                'unknown',
                true,
                true
            )
            expect(mockProduceMessageToDLQ).toHaveBeenCalledWith(
                config.outputs,
                messages[4],
                expect.any(Error),
                'unknown'
            )
        })

        it('counts a timeout result without producing anything', async () => {
            const result = timeout('budget exceeded before slowStep')
            const messages: Message[] = [
                { value: Buffer.from('timed out'), topic: 'test', partition: 0, offset: 1 } as Message,
            ]

            const chunkResults = [createContext(result, { message: messages[0], lastStep: 'slowStep' })]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            expect(results).toHaveLength(1)
            expect(results![0].result).toEqual(result)
            expect(results![0].context.sideEffects).toHaveLength(0)
            expect(mockProduceMessageToDLQ).not.toHaveBeenCalled()
            expect(mockRedirectMessageToOutput).not.toHaveBeenCalled()
            expect(mockLogDroppedMessage).not.toHaveBeenCalled()
            expect(mockIngestionPipelineResultCounter.labels).toHaveBeenCalledWith({
                result: 'timeout',
                last_step_name: 'slowStep',
                details: 'budget exceeded before slowStep',
            })
        })
    })

    describe('last step reporting', () => {
        it('should report last steps for all results including failed messages', async () => {
            const messages: Message[] = [
                { value: Buffer.from('success'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('drop'), topic: 'test', partition: 0, offset: 2 } as Message,
                { value: Buffer.from('redirect'), topic: 'test', partition: 0, offset: 3 } as Message,
                { value: Buffer.from('dlq'), topic: 'test', partition: 0, offset: 4 } as Message,
            ]

            // Create chunk results with different lastStep values
            const chunkResults = [
                createContext(ok({ processed: 'success' }), { message: messages[0], lastStep: 'validationStep' }),
                createContext(drop('dropped item'), { message: messages[1], lastStep: 'filterStep' }),
                createContext(redirect('redirected item', OVERFLOW_OUTPUT), {
                    message: messages[2],
                    lastStep: 'routingStep',
                }),
                createContext(dlq('dlq item', new Error('processing error')), {
                    message: messages[3],
                    lastStep: 'processingStep',
                }),
            ]

            const mockPipeline = createMockPipeline<any, { message: Message }, OverflowOutput>(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return all results including non-success ones
            expect(results).not.toBeNull()
            expect(results).toHaveLength(4)

            // Verify all results were recorded with correct step names
            expect(mockIngestionPipelineResultCounter.labels).toHaveBeenCalledWith({
                result: 'ok',
                last_step_name: 'validationStep',
                details: '',
            })
            expect(mockIngestionPipelineResultCounter.labels).toHaveBeenCalledWith({
                result: 'drop',
                last_step_name: 'filterStep',
                details: 'dropped item',
            })
            expect(mockIngestionPipelineResultCounter.labels).toHaveBeenCalledWith({
                result: 'redirect',
                last_step_name: 'routingStep',
                details: 'overflow(preserve_key=true)',
            })
            expect(mockIngestionPipelineResultCounter.labels).toHaveBeenCalledWith({
                result: 'dlq',
                last_step_name: 'processingStep',
                details: 'dlq item',
            })
        })

        it('should not report last step when context has no lastStep', async () => {
            const messages: Message[] = [
                { value: Buffer.from('success'), topic: 'test', partition: 0, offset: 1 } as Message,
            ]

            // Create chunk results without lastStep
            const chunkResults = [createContext(ok({ processed: 'success' }), { message: messages[0] })]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            expect(results).not.toBeNull()
            expect(results).toEqual([createContext(ok({ processed: 'success' }), { message: messages[0] })])

            // Verify result was recorded with 'unknown' step name
            expect(mockIngestionPipelineResultCounter.labels).toHaveBeenCalledWith({
                result: 'ok',
                last_step_name: 'unknown',
                details: '',
            })
        })
    })

    describe('concurrent processing', () => {
        it('should handle concurrent processing results', async () => {
            const messages: Message[] = [
                { value: Buffer.from('1'), topic: 'test', partition: 0, offset: 1 } as Message,
                { value: Buffer.from('2'), topic: 'test', partition: 0, offset: 2 } as Message,
                { value: Buffer.from('3'), topic: 'test', partition: 0, offset: 3 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [
                createContext(ok({ count: 2 }), { message: messages[0] }),
                createContext(ok({ count: 4 }), { message: messages[1] }),
                createContext(ok({ count: 6 }), { message: messages[2] }),
            ]

            const mockPipeline = createMockPipeline(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            expect(results).not.toBeNull()
            expect(results).toEqual([
                createContext(ok({ count: 2 }), { message: messages[0] }),
                createContext(ok({ count: 4 }), { message: messages[1] }),
                createContext(ok({ count: 6 }), { message: messages[2] }),
            ])
        })
    })

    describe('redirect result with default parameters', () => {
        it('should use default preserveKey and awaitAck when not specified', async () => {
            mockRedirectMessageToOutput.mockResolvedValue(undefined)

            const messages: Message[] = [
                { value: Buffer.from('redirect'), topic: 'test', partition: 0, offset: 1 } as Message,
            ]

            // Create chunk results directly
            const chunkResults = [createContext(redirect('test redirect', OVERFLOW_OUTPUT), { message: messages[0] })]

            const mockPipeline = createMockPipeline<any, { message: Message }, OverflowOutput>(chunkResults)
            const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
            const results = await resultPipeline.next()

            // Should return the redirect result with side effects
            expect(results).not.toBeNull()
            expect(results).toHaveLength(1)
            expect(results![0].result).toEqual(redirect('test redirect', OVERFLOW_OUTPUT))
            expect(results![0].context.sideEffects).toHaveLength(1)

            // Extract and await side effects
            const sideEffects = results![0].context.sideEffects
            await Promise.all(sideEffects)

            // Now verify the mock was called with default parameters
            expect(mockRedirectMessageToOutput).toHaveBeenCalledWith(
                config.outputs,
                OVERFLOW_OUTPUT,
                mockPromiseScheduler,
                messages[0],
                'unknown',
                true, // default preserveKey
                true // default awaitAck
            )
        })
    })
})

describe('Integration tests', () => {
    let mockPromiseScheduler: PromiseScheduler
    let config: PipelineConfig<OverflowOutput>

    beforeEach(() => {
        jest.clearAllMocks()

        mockPromiseScheduler = {
            schedule: jest.fn(),
        } as unknown as PromiseScheduler

        config = {
            outputs: createMockIngestionOutputs<
                typeof DLQ_OUTPUT | typeof OVERFLOW_OUTPUT | typeof INGESTION_WARNINGS_OUTPUT
            >(),
            promiseScheduler: mockPromiseScheduler,
        }
    })

    it('should handle realistic event processing pipeline', async () => {
        const messages: Message[] = [
            { value: Buffer.from('test-event'), topic: 'test', partition: 0, offset: 1 } as Message,
        ]

        // Create chunk results directly
        const chunkResults = [
            createContext(
                ok({
                    eventType: 'pageview',
                    userId: 'user123',
                    isValid: true,
                    timestamp: '2023-01-01T00:00:00Z',
                }),
                { message: messages[0] }
            ),
        ]

        const mockPipeline = createMockPipeline(chunkResults)
        const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
        const results = await resultPipeline.next()

        expect(results).toEqual([
            createContext(
                ok({
                    eventType: 'pageview',
                    userId: 'user123',
                    isValid: true,
                    timestamp: '2023-01-01T00:00:00Z',
                }),
                { message: messages[0] }
            ),
        ])
    })

    it('should handle pipeline failure at different stages', async () => {
        const messages: Message[] = [{ value: Buffer.from('test'), topic: 'test', partition: 0, offset: 1 } as Message]

        // Create chunk results directly
        const chunkResults = [
            createContext(dlq('Validation failed', new Error('Invalid data')), { message: messages[0] }),
        ]

        const mockPipeline = createMockPipeline(chunkResults)
        const resultPipeline = new ResultHandlingPipeline(mockPipeline, config)
        const results = await resultPipeline.next()

        // Should return the DLQ result with side effects
        expect(results).toEqual([
            createContext(dlq('Validation failed', new Error('Invalid data')), {
                message: messages[0],
                sideEffects: expect.arrayContaining([expect.any(Promise)]),
            }),
        ])

        // Await the side effects to trigger the DLQ operation
        if (results && results.length > 0) {
            const sideEffects = results[0].context.sideEffects
            await Promise.all(sideEffects)
        }

        expect(mockProduceMessageToDLQ).toHaveBeenCalledWith(config.outputs, messages[0], expect.any(Error), 'unknown')
    })
})