Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.4 KB
Raw Blame
/**
 * # Chapter 6: Gathering Results
 *
 * The `gather()` method collects all results from a streaming pipeline into
 * a single chunk. This is useful when you need all results together before
 * continuing to the next step.
 *
 * ## When to Use gather()
 *
 * - After `concurrently()`: collect all concurrent results into one chunk
 * - After `concurrentlyPerGroup()`: collect all group results into one chunk
 *
 * ## How gather() Works
 *
 * Without gather(), pipelines stream results as they become available:
 * - `concurrently()` returns items one at a time (in input order)
 * - `concurrentlyPerGroup()` returns groups one at a time as each completes
 *
 * With gather(), all results are collected and returned in a single chunk.
 */
import { newChunkPipelineBuilder } from '~/ingestion/framework/builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'

import { collectChunks } from './helpers'

interface Event {
    userId: string
    eventId: number
}

describe('Gathering Results', () => {
    beforeEach(() => {
        jest.useFakeTimers()
    })

    afterEach(() => {
        jest.useRealTimers()
    })

    /**
     * Without gather(), concurrently() returns items one at a time.
     * With gather(), all items are collected into a single chunk.
     * In both cases, input order is preserved.
     *
     * This test uses variable delays to demonstrate ordering:
     * - Item 1: 30ms delay (finishes last)
     * - Item 2: 10ms delay (finishes first)
     * - Item 3: 20ms delay (finishes second)
     */
    it('gather() after concurrently() collects items into one chunk', async () => {
        const delays: Record<number, number> = { 1: 30, 2: 10, 3: 20 }

        function createVariableDelayStep(): ProcessingStep<number, number> {
            return async function variableDelayStep(n) {
                await new Promise((resolve) => setTimeout(resolve, delays[n]))
                return ok(n * 10)
            }
        }

        // Without gather: items stream one at a time in input order
        const streamingPipeline = newChunkPipelineBuilder<number>()
            .concurrently((builder) => builder.pipe(createVariableDelayStep()))
            .build()

        streamingPipeline.feed([1, 2, 3].map((n) => createOkContext(n, {})))
        const streamingPromise = collectChunks(streamingPipeline)
        await jest.advanceTimersByTimeAsync(30)
        const streamingChunks = await streamingPromise

        // Without gather: 3 separate chunks, one item each, in input order
        expect(streamingChunks).toEqual([[10], [20], [30]])

        // With gather: all items collected into one chunk
        const gatheringPipeline = newChunkPipelineBuilder<number>()
            .concurrently((builder) => builder.pipe(createVariableDelayStep()))
            .gather()
            .build()

        gatheringPipeline.feed([1, 2, 3].map((n) => createOkContext(n, {})))
        const gatheringPromise = collectChunks(gatheringPipeline)
        await jest.advanceTimersByTimeAsync(30)
        const gatheringChunks = await gatheringPromise

        // With gather: one chunk with all items, input order preserved
        expect(gatheringChunks).toEqual([[10, 20, 30]])
    })

    /**
     * Without gather(), concurrentlyPerGroup() returns groups one at a time
     * as each group completes. With gather(), all groups are collected into
     * a single chunk.
     *
     * Within each group, events are always processed sequentially (order preserved).
     */
    it('gather() after concurrentlyPerGroup() collects groups into one chunk', async () => {
        const delays: Record<string, number> = {
            alice: 30,
            bob: 10,
        }

        function createVariableDelayStep(): ProcessingStep<Event, Event> {
            return async function variableDelayStep(event) {
                await new Promise((resolve) => setTimeout(resolve, delays[event.userId]))
                return ok(event)
            }
        }

        // Input order: alice:1, bob:2, alice:3, bob:4
        const events: Event[] = [
            { userId: 'alice', eventId: 1 },
            { userId: 'bob', eventId: 2 },
            { userId: 'alice', eventId: 3 },
            { userId: 'bob', eventId: 4 },
        ]

        // Without gather: groups stream one at a time in completion order
        const streamingPipeline = newChunkPipelineBuilder<Event>()
            .concurrentlyPerGroup(
                (event) => event.userId,
                (group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createVariableDelayStep()))
            )
            .build()

        streamingPipeline.feed(events.map((e) => createOkContext(e, {})))
        // bob: 2 events x 10ms = 20ms, alice: 2 events x 30ms = 60ms
        const streamingPromise = collectChunks(streamingPipeline)
        await jest.advanceTimersByTimeAsync(60)
        const streamingChunks = await streamingPromise

        // Without gather: bob's group finishes first, then alice's
        // Within each group, events are in order (2 before 4, 1 before 3)
        expect(streamingChunks).toEqual([
            [
                { userId: 'bob', eventId: 2 },
                { userId: 'bob', eventId: 4 },
            ],
            [
                { userId: 'alice', eventId: 1 },
                { userId: 'alice', eventId: 3 },
            ],
        ])

        // With gather: all groups collected into one chunk
        const gatheringPipeline = newChunkPipelineBuilder<Event>()
            .concurrentlyPerGroup(
                (event) => event.userId,
                (group) => group.sequentially((groupBuilder) => groupBuilder.pipe(createVariableDelayStep()))
            )
            .gather()
            .build()

        gatheringPipeline.feed(events.map((e) => createOkContext(e, {})))
        const gatheringPromise = collectChunks(gatheringPipeline)
        await jest.advanceTimersByTimeAsync(60)
        const gatheringChunks = await gatheringPromise

        // With gather: one chunk with all items
        // Groups are in completion order (bob first), within-group order preserved
        expect(gatheringChunks).toEqual([
            [
                { userId: 'bob', eventId: 2 },
                { userId: 'bob', eventId: 4 },
                { userId: 'alice', eventId: 1 },
                { userId: 'alice', eventId: 3 },
            ],
        ])
    })
})