Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.8 KB
Raw Blame
/**
 * # Chapter 4: Sequential Processing
 *
 * The `sequentially()` method processes items one at a time, preserving order.
 * Use it when order matters or when you need to limit concurrent operations.
 * Unlike concurrent processing which returns items one by one as they complete,
 * sequential processing returns all items together in a single chunk after
 * all processing is done.
 *
 * ## When to Use Sequential Processing
 *
 * - Operations that must happen in order (e.g., database transactions)
 * - Rate-limited APIs where you can't make parallel requests
 * - When you need predictable resource usage
 *
 * ## Tradeoffs
 *
 * - **Order**: Strictly preserved
 * - **Throughput**: Lower than concurrent (one at a time)
 * - **Resources**: Minimal concurrent connections
 * - **Chunking**: Results returned together (not streamed)
 */
import { newChunkPipelineBuilder } from '~/ingestion/framework/builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { isOkResult, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'

import { consumeAll } from './helpers'

describe('Sequential Processing', () => {
    beforeEach(() => {
        jest.useFakeTimers()
    })

    afterEach(() => {
        jest.useRealTimers()
    })

    /**
     * The sequentially() method processes items one at a time.
     * Each item waits for the previous one to complete.
     */
    it('sequentially() processes items one at a time', async () => {
        const processingOrder: number[] = []

        function createOrderTrackingStep(): ProcessingStep<number, number> {
            return async function orderTrackingStep(n) {
                await new Promise((resolve) => setTimeout(resolve, 10))
                processingOrder.push(n)
                return ok(n)
            }
        }

        const pipeline = newChunkPipelineBuilder<number>()
            .sequentially((builder) => builder.pipe(createOrderTrackingStep()))
            .build()

        const batch = [1, 2, 3].map((n) => createOkContext(n, {}))
        pipeline.feed(batch)

        // Sequential: need to advance time for each item (3 items x 10ms = 30ms)
        const allValues = await consumeAll(pipeline, 30)

        // Items processed in order
        expect(processingOrder).toEqual([1, 2, 3])
        // Results also in order
        expect(allValues).toEqual([1, 2, 3])
    })

    /**
     * Unlike concurrent processing, sequential returns all items in one chunk
     * after processing them one at a time.
     */
    it('all items are returned together after sequential processing', async () => {
        function createSlowStep(): ProcessingStep<number, number> {
            return async function slowStep(n) {
                await new Promise((resolve) => setTimeout(resolve, 10))
                return ok(n * 10)
            }
        }

        const pipeline = newChunkPipelineBuilder<number>()
            .sequentially((builder) => builder.pipe(createSlowStep()))
            .build()

        const batch = [1, 2, 3].map((n) => createOkContext(n, {}))
        pipeline.feed(batch)

        // Collect chunks as they arrive
        const chunks: number[][] = []
        const collectResults = (async () => {
            let result = await pipeline.next()
            while (result !== null) {
                chunks.push(
                    result.filter((r) => isOkResult(r.result)).map((r) => (r.result as { value: number }).value)
                )
                result = await pipeline.next()
            }
        })()

        // Sequential: need 30ms total (10ms per item)
        await jest.advanceTimersByTimeAsync(30)
        await collectResults

        // All items returned in a single chunk (unlike concurrent which returns 1 at a time)
        expect(chunks).toEqual([[10, 20, 30]])
    })

    /**
     * Result ordering always matches input order with sequential processing.
     */
    it('result ordering matches input order', async () => {
        const completionOrder: number[] = []

        function createTrackingStep(): ProcessingStep<number, number> {
            return async function trackingStep(n) {
                await new Promise((resolve) => setTimeout(resolve, 10))
                completionOrder.push(n)
                return ok(n * 10)
            }
        }

        const pipeline = newChunkPipelineBuilder<number>()
            .sequentially((builder) => builder.pipe(createTrackingStep()))
            .build()

        const batch = [3, 1, 2].map((n) => createOkContext(n, {}))
        pipeline.feed(batch)

        const allValues = await consumeAll(pipeline, 30)

        // Items processed in input order (3, 1, 2)
        expect(completionOrder).toEqual([3, 1, 2])

        // Results also in input order (3, 1, 2 -> 30, 10, 20)
        expect(allValues).toEqual([30, 10, 20])
    })
})