Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.7 KB
Raw Blame
/**
 * # Chapter 15: Consuming a Pipeline
 *
 * The earlier chapters build pipelines; this one shows how a built
 * `ChunkPipeline` is actually driven in production, and how
 * `ChunkPipelineUnwrapper` turns its results into plain values.
 *
 * ## The feed() / next() contract
 *
 * A `ChunkPipeline` is a pull-based stream:
 *
 * - **`feed(elements)`** hands a batch of `OkResultWithContext` items to the
 *   pipeline. It buffers them; it does not process synchronously.
 * - **`next()`** pulls the next available chunk of results (each a
 *   result-with-context), driving processing as needed. Depending on the
 *   stages involved, a `next()` may return one item, one group, or a whole
 *   batch.
 * - **`next()` returns `null` when the pipeline is drained** - everything fed
 *   has been processed and emitted. A consumer loops `next()` until it sees
 *   `null`, then feeds the next batch.
 *
 * ```
 * pipeline.feed(batch)
 * let results = await pipeline.next()
 * while (results !== null) {
 *   // handle results
 *   results = await pipeline.next()
 * }
 * ```
 *
 * ## ChunkPipelineUnwrapper
 *
 * Most consumers do not want result-with-context objects; they want the plain
 * output values. `ChunkPipelineUnwrapper` wraps a pipeline and, on `next()`,
 * returns a flat `TOutput[]` of just the OK values - non-OK results (DLQ, DROP,
 * REDIRECT, TIMEOUT) are filtered out (they are assumed already handled by a
 * `handleResults()` stage upstream). It returns `null` when the underlying
 * pipeline drains.
 *
 * ### Warning on unhandled side effects
 *
 * The unwrapper is a terminal consumer, so any side effects still attached to
 * the results it sees will never be awaited. When it finds results that still
 * carry side effects, it logs a warning - a signal that a `handleSideEffects()`
 * stage is missing from the pipeline.
 */
import { logger } from '~/common/utils/logger'
import { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'
import { newChunkPipelineBuilder } from '~/ingestion/framework/builders'
import { ChunkPipelineUnwrapper } from '~/ingestion/framework/chunk-pipeline-unwrapper'
import { createOkContext } from '~/ingestion/framework/helpers'
import { drop, isOkResult, ok } from '~/ingestion/framework/results'

interface Event {
    id: number
}

type NoCtx = Record<string, never>

describe('Consuming a Pipeline', () => {
    /**
     * A consumer feeds a batch, then loops next() until it returns null.
     * next() drives processing; null marks the pipeline drained.
     */
    it('feed() then next() drives the pipeline until it returns null', async () => {
        function createDoubleStep(): ChunkProcessingStep<Event, Event> {
            return function doubleStep(events) {
                return Promise.resolve(events.map((event) => ok({ id: event.id * 2 })))
            }
        }

        const pipeline = newChunkPipelineBuilder<Event, NoCtx>().pipeChunk(createDoubleStep()).build()

        pipeline.feed([{ id: 1 }, { id: 2 }, { id: 3 }].map((e) => createOkContext(e, {})))

        const values: number[] = []
        let results = await pipeline.next()
        while (results !== null) {
            for (const r of results) {
                if (isOkResult(r.result)) {
                    values.push(r.result.value.id)
                }
            }
            results = await pipeline.next()
        }

        expect(values).toEqual([2, 4, 6])
        // Once drained, next() keeps returning null
        expect(await pipeline.next()).toBeNull()
    })

    /**
     * ChunkPipelineUnwrapper returns plain output values and filters out non-OK
     * results (here, DROP). The consumer never sees result-with-context objects.
     */
    it('ChunkPipelineUnwrapper returns plain values and drops non-OK results', async () => {
        function createFilterStep(): ChunkProcessingStep<Event, Event> {
            return function filterStep(events) {
                // Drop even ids, keep odd ones
                return Promise.resolve(events.map((event) => (event.id % 2 === 0 ? drop('even filtered') : ok(event))))
            }
        }

        const pipeline = newChunkPipelineBuilder<Event, NoCtx>().pipeChunk(createFilterStep()).build()
        const unwrapper = new ChunkPipelineUnwrapper(pipeline)

        unwrapper.feed([{ id: 1 }, { id: 2 }, { id: 3 }].map((e) => createOkContext(e, {})))

        const values: Event[] = []
        let chunk = await unwrapper.next()
        while (chunk !== null) {
            values.push(...chunk)
            chunk = await unwrapper.next()
        }

        // Only the OK (odd) values survive, as plain objects
        expect(values).toEqual([{ id: 1 }, { id: 3 }])
    })

    /**
     * When results still carry side effects (no handleSideEffects() stage ran),
     * the unwrapper logs a warning: those side effects would otherwise be
     * silently dropped.
     */
    it('ChunkPipelineUnwrapper warns about unhandled side effects', async () => {
        const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined)

        function createStepWithSideEffect(): ChunkProcessingStep<Event, Event> {
            return function stepWithSideEffect(events) {
                // Attach a side effect that no downstream stage handles
                return Promise.resolve(events.map((event) => ok(event, [Promise.resolve('unhandled')])))
            }
        }

        const pipeline = newChunkPipelineBuilder<Event, NoCtx>().pipeChunk(createStepWithSideEffect()).build()
        const unwrapper = new ChunkPipelineUnwrapper(pipeline)

        unwrapper.feed([{ id: 1 }].map((e) => createOkContext(e, {})))
        await unwrapper.next()

        expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('remaining side effects'))

        warnSpy.mockRestore()
    })
})