Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.9 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { BufferingChunkPipeline } from './buffering-chunk-pipeline'
import { DefaultContext, createContext, createOkContext } from './helpers'
import { ok } from './results'

describe('BufferingChunkPipeline', () => {
    let message1: Message
    let message2: Message
    let message3: Message
    let context1: { message: Message }
    let context2: { message: Message }
    let context3: { message: Message }

    beforeEach(() => {
        // Create different mock messages with unique properties
        message1 = {
            topic: 'test-topic',
            partition: 0,
            offset: 1,
            key: Buffer.from('key1'),
            value: Buffer.from('value1'),
            timestamp: Date.now(),
        } as Message

        message2 = {
            topic: 'test-topic',
            partition: 0,
            offset: 2,
            key: Buffer.from('key2'),
            value: Buffer.from('value2'),
            timestamp: Date.now() + 1,
        } as Message

        message3 = {
            topic: 'test-topic',
            partition: 0,
            offset: 3,
            key: Buffer.from('key3'),
            value: Buffer.from('value3'),
            timestamp: Date.now() + 2,
        } as Message

        context1 = { message: message1 }
        context2 = { message: message2 }
        context3 = { message: message3 }
    })

    describe('constructor', () => {
        it('should create instance with default type', () => {
            const pipeline = new BufferingChunkPipeline()
            expect(pipeline).toBeInstanceOf(BufferingChunkPipeline)
        })

        it('should create instance with custom type', () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            expect(pipeline).toBeInstanceOf(BufferingChunkPipeline)
        })
    })

    describe('feed', () => {
        it('should add elements to buffer', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const batch = [createOkContext('hello', context1), createOkContext('world', context2)]

            pipeline.feed(batch)

            // Buffer is internal, so we test through next()
            const result = await pipeline.next()
            expect(result).toEqual([createContext(ok('hello'), context1), createContext(ok('world'), context2)])
        })

        it('should accumulate multiple feeds', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const batch1 = [createOkContext('hello', context1)]
            const batch2 = [createOkContext('world', context2)]

            pipeline.feed(batch1)
            pipeline.feed(batch2)

            const result = await pipeline.next()
            expect(result).toEqual([createContext(ok('hello'), context1), createContext(ok('world'), context2)])
        })

        it('should handle empty batch', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const emptyBatch: [] = []

            pipeline.feed(emptyBatch)

            const result = await pipeline.next()
            expect(result).toEqual(null)
        })
    })

    describe('next', () => {
        it('should return null when buffer is empty', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const result = await pipeline.next()
            expect(result).toBeNull()
        })

        it('should return all buffered elements and clear buffer', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const batch = [createOkContext('hello', context1), createOkContext('world', context2)]

            pipeline.feed(batch)

            const result1 = await pipeline.next()
            const result2 = await pipeline.next()

            expect(result1).toEqual([createContext(ok('hello'), context1), createContext(ok('world'), context2)])
            expect(result2).toBeNull()
        })

        it('should handle mixed result types', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()

            const batch = [createOkContext('hello', context1)]

            pipeline.feed(batch)

            const result = await pipeline.next()
            const result2 = await pipeline.next()

            expect(result).toEqual([createContext(ok('hello'), context1)])
            expect(result2).toBeNull()
        })

        it('should preserve order of fed elements', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const batch1 = [createOkContext('first', context1)]
            const batch2 = [createOkContext('second', context2)]
            const batch3 = [createOkContext('third', context3)]

            pipeline.feed(batch1)
            pipeline.feed(batch2)
            pipeline.feed(batch3)

            const result = await pipeline.next()
            const result2 = await pipeline.next()

            expect(result).toEqual([
                createContext(ok('first'), context1),
                createContext(ok('second'), context2),
                createContext(ok('third'), context3),
            ])
            expect(result2).toBeNull()
        })

        it('should handle large number of elements', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()
            const batch = []

            for (let i = 0; i < 100; i++) {
                batch.push(createOkContext(`item${i}`, context1))
            }

            pipeline.feed(batch)

            const result = await pipeline.next()
            const result2 = await pipeline.next()

            expect(result).toHaveLength(100)
            expect(result![0]).toEqual(createContext(ok('item0'), context1))
            expect(result![99]).toEqual(createContext(ok('item99'), context1))
            expect(result2).toBeNull()
        })

        it('should resume after returning null when more elements are fed', async () => {
            const pipeline = new BufferingChunkPipeline<string, DefaultContext>()

            // First round: feed and process
            const batch1 = [createOkContext('first', context1)]
            pipeline.feed(batch1)

            const result1 = await pipeline.next()
            expect(result1).toEqual([createContext(ok('first'), context1)])

            // Should return null when buffer is empty
            const result2 = await pipeline.next()
            expect(result2).toBeNull()

            // Feed more elements
            const batch2 = [createOkContext('second', context2)]
            pipeline.feed(batch2)

            // Should resume processing
            const result3 = await pipeline.next()
            expect(result3).toEqual([createContext(ok('second'), context2)])

            // Should return null again
            const result4 = await pipeline.next()
            expect(result4).toBeNull()
        })
    })
})