Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.6 KB
Raw Blame
import { BranchDecisionFn, BranchingPipeline } from '~/ingestion/framework/branching-pipeline'
import { Pipeline } from '~/ingestion/framework/pipeline.interface'
import { RetryOptions, withStepRetry } from '~/ingestion/framework/retry'
import {
    PromiseSchedulerInterface,
    SideEffectHandlingConfig,
    SideEffectHandlingProcessor,
} from '~/ingestion/framework/side-effect-handling-pipeline'
import { StartPipeline } from '~/ingestion/framework/start-pipeline'
import { StepPipeline } from '~/ingestion/framework/step-pipeline'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { TimeoutBarrierPipeline } from '~/ingestion/framework/timeout-barrier-pipeline'

export class StartPipelineBuilder<T, C> {
    pipe<U, R extends string = never>(
        step: ProcessingStep<T, U, R>,
        options?: { retry?: RetryOptions }
    ): PipelineBuilder<T, U, C, R> {
        const wrappedStep = options?.retry ? withStepRetry(step, options.retry) : step
        return new PipelineBuilder(new StepPipeline(wrappedStep, new StartPipeline<T, C>()))
    }

    branching<TBranch extends string, U, RBranch extends string = never>(
        decisionFn: BranchDecisionFn<T, TBranch>,
        callback: (
            builder: BranchingPipelineBuilder<T, T, U, C, TBranch, TBranch>
        ) => BranchingPipelineBuilder<T, T, U, C, TBranch, never, never, RBranch>
    ): PipelineBuilder<T, U, C, RBranch> {
        const branchingBuilder = new BranchingPipelineBuilder<T, T, U, C, TBranch, TBranch>(
            decisionFn,
            new StartPipeline<T, C>()
        )
        const finalBuilder = callback(branchingBuilder)
        return new PipelineBuilder(finalBuilder.build())
    }
}

export class PipelineBuilder<TInput, TOutput, C, R extends string = never> {
    constructor(protected pipeline: Pipeline<TInput, TOutput, C, R>) {}

    pipe<U, R2 extends string = never>(
        step: ProcessingStep<TOutput, U, R2>,
        options?: { retry?: RetryOptions }
    ): PipelineBuilder<TInput, U, C, R | R2> {
        const wrappedStep = options?.retry ? withStepRetry(step, options.retry) : step
        return new PipelineBuilder(new StepPipeline(wrappedStep, this.pipeline))
    }

    branching<TBranch extends string, U, RBranch extends string = never>(
        decisionFn: BranchDecisionFn<TOutput, TBranch>,
        callback: (
            builder: BranchingPipelineBuilder<TInput, TOutput, U, C, TBranch, TBranch, R>
        ) => BranchingPipelineBuilder<TInput, TOutput, U, C, TBranch, never, R, RBranch>
    ): PipelineBuilder<TInput, U, C, R | RBranch> {
        const branchingBuilder = new BranchingPipelineBuilder<TInput, TOutput, U, C, TBranch, TBranch, R>(
            decisionFn,
            this.pipeline
        )
        const finalBuilder = callback(branchingBuilder)
        return new PipelineBuilder(finalBuilder.build())
    }

    /** Past this point the budget no longer stops an element. See {@link TimeoutBarrierPipeline}. */
    timeoutBarrier(): PipelineBuilder<TInput, TOutput, C, R> {
        return new PipelineBuilder(new TimeoutBarrierPipeline(this.pipeline))
    }

    /**
     * Schedule (or await) the side effects carried by this pipeline's results
     * and clear them, so downstream consumers never have to drain them.
     * Mirrors `ChunkPipelineBuilder.handleSideEffects`.
     */
    handleSideEffects(
        promiseScheduler: PromiseSchedulerInterface,
        options: SideEffectHandlingConfig
    ): PipelineBuilder<TInput, TOutput, C, R> {
        return new PipelineBuilder(new SideEffectHandlingProcessor(this.pipeline, promiseScheduler, options))
    }

    build(): Pipeline<TInput, TOutput, C, R> {
        return this.pipeline
    }
}

/**
 * Builder for branching pipelines that tracks remaining branches via TRemaining.
 *
 * Each branch() call removes a branch from TRemaining via Exclude.
 * build() requires TRemaining = never, ensuring all branches are supplied at compile time.
 */
export class BranchingPipelineBuilder<
    TInput,
    TIntermediate,
    TOutput,
    C,
    TBranch extends string,
    TRemaining extends TBranch = TBranch,
    RPrev extends string = never,
    RBranch extends string = never,
> {
    constructor(
        private decisionFn: BranchDecisionFn<TIntermediate, TBranch>,
        private previousPipeline: Pipeline<TInput, TIntermediate, C, RPrev>,
        private branches: Partial<Record<TBranch, Pipeline<TIntermediate, TOutput, C, RBranch>>> = {}
    ) {}

    branch<B extends TRemaining, R2 extends string = never>(
        branchName: B,
        callback: (builder: StartPipelineBuilder<TIntermediate, C>) => PipelineBuilder<TIntermediate, TOutput, C, R2>
    ): BranchingPipelineBuilder<
        TInput,
        TIntermediate,
        TOutput,
        C,
        TBranch,
        Exclude<TRemaining, B>,
        RPrev,
        RBranch | R2
    > {
        const branchPipeline = callback(new StartPipelineBuilder<TIntermediate, C>()).build()
        // Split spread and computed key into two statements — the spread preserves
        // covariance (Pipeline is covariant in R), and the key assignment is type-safe.
        const updatedBranches: Partial<Record<TBranch, Pipeline<TIntermediate, TOutput, C, RBranch | R2>>> = {
            ...this.branches,
        }
        updatedBranches[branchName] = branchPipeline
        return new BranchingPipelineBuilder(this.decisionFn, this.previousPipeline, updatedBranches)
    }

    build(
        this: BranchingPipelineBuilder<TInput, TIntermediate, TOutput, C, TBranch, never, RPrev, RBranch>
    ): Pipeline<TInput, TOutput, C, RPrev | RBranch> {
        return new BranchingPipeline(this.decisionFn, this.branches, this.previousPipeline)
    }
}