:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { IngestionWarningsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import {
BaseChunkPipeline,
ChunkProcessingStep,
applyChunkStepToResults,
} from '~/ingestion/framework/base-chunk-pipeline'
import { BufferingChunkPipeline } from '~/ingestion/framework/buffering-chunk-pipeline'
import { ChunkPipeline } from '~/ingestion/framework/chunk-pipeline.interface'
import { ConcurrentChunkProcessingPipeline } from '~/ingestion/framework/concurrent-chunk-pipeline'
import {
ConcurrentlyGroupingChunkPipeline,
GroupChunkProcessor,
GroupingFunction,
processGroupChunkSequentially,
} from '~/ingestion/framework/concurrently-grouping-chunk-pipeline'
import {
FanInFunction,
FanOutFanInChunkPipeline,
FanOutFunction,
FanOutSubContext,
} from '~/ingestion/framework/fan-out-fan-in-chunk-pipeline'
import { FilterMapChunkPipeline, FilterMapMappingFunction } from '~/ingestion/framework/filter-map-chunk-pipeline'
import { GatheringChunkPipeline } from '~/ingestion/framework/gathering-chunk-pipeline'
import { IngestionWarningHandlingChunkPipeline } from '~/ingestion/framework/ingestion-warning-handling-chunk-pipeline'
import { PipelineBuilderContext } from '~/ingestion/framework/pipeline.interface'
import { PipelineConfig, ResultHandlingPipeline } from '~/ingestion/framework/result-handling-pipeline'
import { RetryOptions, withChunkRetry } from '~/ingestion/framework/retry'
import { SequentialChunkPipeline } from '~/ingestion/framework/sequential-chunk-pipeline'
import { SideEffectHandlingPipeline } from '~/ingestion/framework/side-effect-handling-pipeline'
import { PipelineBuilder, StartPipelineBuilder } from './pipeline-builders'
/**
* Minimal team context required for team-aware pipeline operations.
* Only the team ID is needed to route warnings to the correct team.
*/
export interface TeamIdContext {
team: { id: number }
}
/**
* Composes how a group's queued chunk is processed, as a chain of group chunk
* stages: `pipeChunk` applies a chunk step to the whole chunk, `sequentially`
* runs each item through a per-item pipeline one at a time. Stages run in
* declaration order; `concurrentlyPerGroup` finalizes the chain into the
* single {@link GroupChunkProcessor} the grouping pipeline executes per chunk.
*/
export class GroupProcessingBuilder<
TInput,
TOutput,
CInput = Record<string, never>,
COutput = CInput,
R extends string = never,
D = unknown,
> {
// Holds a completion function instead of the grouping function and previous
// pipeline directly: a groupingFn field would be contravariant in TOutput,
// making ChunkPipelineBuilder invariant in TOutput and breaking covariant
// subpipeline callbacks (e.g. in newBatchingPipeline). In this closure's
// signature TOutput only appears in variance-neutral positions.
constructor(
/** @internal Completes the group definition; called by concurrentlyPerGroup with the terminal stage. */
readonly buildGroupedPipeline: <U, R2 extends string>(
downstream: GroupChunkProcessor<TOutput, U, COutput, R2>
) => ChunkPipeline<TInput, U, CInput, COutput, R | R2>,
private builderContext?: PipelineBuilderContext<D>
) {}
private stage<U, R2 extends string>(
stageProcessor: GroupChunkProcessor<TOutput, U, COutput, R2>
): GroupProcessingBuilder<TInput, U, CInput, COutput, R | R2, D> {
const buildWithEarlierStages = this.buildGroupedPipeline
return new GroupProcessingBuilder<TInput, U, CInput, COutput, R | R2, D>(
<V, R3 extends string>(downstream: GroupChunkProcessor<U, V, COutput, R3>) =>
buildWithEarlierStages<V, R2 | R3>(async (items) => await downstream(await stageProcessor(items))),
this.builderContext
)
}
/**
* Apply a chunk step to each group's queued chunk. Same contract as the
* outer `pipeChunk`: the step sees the chunk's OK values in processing
* order and returns one result per value; non-OK results pass through.
* Runs once per started group chunk.
*/
pipeChunk<U, R2 extends string = never>(
step: ChunkProcessingStep<TOutput, U, R2>
): GroupProcessingBuilder<TInput, U, CInput, COutput, R | R2, D> {
const stepName = step.name || 'anonymousChunkStep'
return this.stage((items) => applyChunkStepToResults(step, stepName, items, this.builderContext))
}
/**
* Process the items within each group sequentially through the provided
* pipeline: one item at a time, in input order. Groups still run
* concurrently with respect to each other.
*/
sequentially<U, R2 extends string = never>(
callback: (builder: StartPipelineBuilder<TOutput, COutput>) => PipelineBuilder<TOutput, U, COutput, R2>
): GroupProcessingBuilder<TInput, U, CInput, COutput, R | R2, D> {
const processor = callback(new StartPipelineBuilder<TOutput, COutput>()).build()
return this.stage(processGroupChunkSequentially(processor))
}
}
/**
* Middle stage of `.fanOut(fn).via(cb).fanIn(fn)`. Its only method is `via`,
* so a fan-out stage cannot be built (or have results handled) until it is
* closed with a subpipeline and a fan-in function.
*/
export class FanOutBuilder<
TInput,
TOutput,
TSub,
CInput = Record<string, never>,
COutput = CInput,
R extends string = never,
D = unknown,
> {
// Holds a completion function instead of the fan-out function and previous
// pipeline directly, for the same variance reason as GroupProcessingBuilder:
// in this closure's signature TOutput only appears in variance-neutral
// positions, keeping ChunkPipelineBuilder covariant in TOutput.
constructor(
private readonly buildFannedOutPipeline: <TSubOut, U, RSub extends string>(
subPipeline: ChunkPipeline<TSub, TSubOut, FanOutSubContext, FanOutSubContext, RSub>,
fanInFn: FanInFunction<TOutput, TSubOut, U>
) => ChunkPipeline<TInput, U, CInput, COutput, R>,
private readonly builderContext?: PipelineBuilderContext<D>
) {}
/**
* Route the sub-elements through a subpipeline built on the full chunk
* builder surface (`concurrently` with `maxConcurrency`, per-step `retry`,
* `concurrentlyPerGroup`, …). Sub-elements from all parents share the
* subpipeline, so one concurrency cap governs the whole stage.
*
* Redirect sub-results never escape the stage, so the subpipeline's
* redirect names (`RSub`) do not propagate to the stage's result type —
* downstream `handleResults` won't demand outputs for redirects that
* cannot happen. (A DLQ sub-result does surface, as the parent's
* aggregated DLQ, which carries no redirect names.)
*/
via<TSubOut, RSub extends string = never>(
subpipelineCallback: (
builder: ChunkPipelineBuilder<TSub, TSub, FanOutSubContext, FanOutSubContext, never, D>
) => ChunkPipelineBuilder<TSub, TSubOut, FanOutSubContext, FanOutSubContext, RSub, D>
): FanInBuilder<TInput, TOutput, TSubOut, CInput, COutput, R, D> {
// Sub-contexts carry a copy of their parent's debugContext, so the
// builder context (and its aggregator) applies inside the subpipeline
// as-is.
const startBuilder = new ChunkPipelineBuilder<TSub, TSub, FanOutSubContext, FanOutSubContext, never, D>(
new BufferingChunkPipeline<TSub, FanOutSubContext>(),
this.builderContext
)
const subPipeline = subpipelineCallback(startBuilder).build()
return new FanInBuilder(
<U>(fanInFn: FanInFunction<TOutput, TSubOut, U>) => this.buildFannedOutPipeline(subPipeline, fanInFn),
this.builderContext
)
}
}
/**
* Final stage of `.fanOut(fn).via(cb).fanIn(fn)`. Its only method is `fanIn`,
* which folds the sub-results back into the parent and returns a regular
* {@link ChunkPipelineBuilder}.
*/
export class FanInBuilder<
TInput,
TOutput,
TSubOut,
CInput = Record<string, never>,
COutput = CInput,
R extends string = never,
D = unknown,
> {
// Completion-closure shape for the same variance reason as FanOutBuilder.
constructor(
private readonly buildFannedInPipeline: <U>(
fanInFn: FanInFunction<TOutput, TSubOut, U>
) => ChunkPipeline<TInput, U, CInput, COutput, R>,
private readonly builderContext?: PipelineBuilderContext<D>
) {}
/**
* Close the stage: fold each parent's collected sub-results back into the
* original element (synchronous, cheap).
*/
fanIn<U>(fanInFn: FanInFunction<TOutput, TSubOut, U>): ChunkPipelineBuilder<TInput, U, CInput, COutput, R, D> {
return new ChunkPipelineBuilder(this.buildFannedInPipeline(fanInFn), this.builderContext)
}
}
export class ChunkPipelineBuilder<TInput, TOutput, CInput, COutput = CInput, R extends string = never, D = unknown> {
constructor(
protected pipeline: ChunkPipeline<TInput, TOutput, CInput, COutput, R>,
protected builderContext?: PipelineBuilderContext<D>
) {}
pipeChunk<U, R2 extends string = never>(
step: ChunkProcessingStep<TOutput, U, R2>,
options?: { retry?: RetryOptions }
): ChunkPipelineBuilder<TInput, U, CInput, COutput, R | R2, D> {
const wrappedStep = options?.retry ? withChunkRetry(step, options.retry) : step
return new ChunkPipelineBuilder(
new BaseChunkPipeline(wrappedStep, this.pipeline, this.builderContext),
this.builderContext
)
}
/**
* Process each item of the chunk concurrently, emitting results in input (FIFO) order.
*
* @param options.maxConcurrency - Cap on how many items process at once. Omitted means unbounded.
*/
concurrently<U, R2 extends string = never>(
callback: (builder: StartPipelineBuilder<TOutput, COutput>) => PipelineBuilder<TOutput, U, COutput, R2>,
options?: { maxConcurrency?: number }
): ChunkPipelineBuilder<TInput, U, CInput, COutput, R | R2, D> {
const processor = callback(new StartPipelineBuilder<TOutput, COutput>()).build()
return new ChunkPipelineBuilder(
new ConcurrentChunkProcessingPipeline(processor, this.pipeline, options?.maxConcurrency),
this.builderContext
)
}
sequentially<U, R2 extends string = never>(
callback: (builder: StartPipelineBuilder<TOutput, COutput>) => PipelineBuilder<TOutput, U, COutput, R2>
): ChunkPipelineBuilder<TInput, U, CInput, COutput, R | R2, D> {
const processor = callback(new StartPipelineBuilder<TOutput, COutput>()).build()
return new ChunkPipelineBuilder(new SequentialChunkPipeline(processor, this.pipeline), this.builderContext)
}
gather(): ChunkPipelineBuilder<TInput, TOutput, CInput, COutput, R, D> {
return new ChunkPipelineBuilder(new GatheringChunkPipeline(this.pipeline), this.builderContext)
}
/**
* Filters OK results, applies a mapping function, and processes through a subpipeline.
* Non-OK results are passed through unchanged.
*
* @param mappingFn - Function to map OK results (transforms both value and context)
* @param subpipelineCallback - Callback that receives a builder and returns the subpipeline
*/
filterMap<TMapped, TSubOutput, CMapped = COutput, CSubOutput = CMapped, ROut extends string = never>(
mappingFn: FilterMapMappingFunction<TOutput, TMapped, COutput, CMapped>,
subpipelineCallback: (
builder: ChunkPipelineBuilder<TMapped, TMapped, CMapped, CMapped, never, D>
) => ChunkPipelineBuilder<TMapped, TSubOutput, CMapped, CSubOutput, ROut, D>
): ChunkPipelineBuilder<TInput, TSubOutput, CInput, CSubOutput | COutput, R | ROut, D> {
const startBuilder = new ChunkPipelineBuilder<TMapped, TMapped, CMapped, CMapped, never, D>(
new BufferingChunkPipeline<TMapped, CMapped>(),
this.builderContext
)
const subpipelineBuilder = subpipelineCallback(startBuilder)
const subPipeline = subpipelineBuilder.build()
return new ChunkPipelineBuilder(
new FilterMapChunkPipeline<
TInput,
TOutput,
TMapped,
TSubOutput,
CInput,
COutput,
CMapped,
CSubOutput,
R,
ROut
>(this.pipeline, mappingFn, subPipeline),
this.builderContext
)
}
/**
* Open a fan-out/fan-in stage: fan each OK element out into sub-elements,
* process them through a subpipeline, and fan the results back into the
* original element. The stage is staged so each part reads at the call
* site — `.fanOut(fn).via((sub) => …).fanIn(fn)` — and only `.fanIn()`
* closes it into a buildable pipeline.
*
* Cardinality is preserved at the parent level (N in, N out); parents emit
* as they complete (unordered). Non-OK elements pass through unchanged.
* OK sub-results are collected for the fan-in; dropped sub-elements are
* silently excluded (DROP is the sanctioned way for a sub-step to discard
* its sub-element); a DLQ sub-result fails the whole parent, which emits a
* DLQ aggregating its sub DLQs instead of fanning in; REDIRECT sub-results
* are excluded with a warning log — sub-elements are not Kafka messages.
*
* Like processing steps, the fan-out and fan-in functions are named
* functions (defined in step files, created by factories where they need
* config) — their `.name` is used for error attribution.
*
* @param fanOutFn - Splits an element into sub-elements (synchronous, cheap)
*/
fanOut<TSub>(fanOutFn: FanOutFunction<TOutput, TSub>): FanOutBuilder<TInput, TOutput, TSub, CInput, COutput, R, D> {
return new FanOutBuilder(
<TSubOut, U, RSub extends string>(
subPipeline: ChunkPipeline<TSub, TSubOut, FanOutSubContext, FanOutSubContext, RSub>,
fanInFn: FanInFunction<TOutput, TSubOut, U>
) =>
new FanOutFanInChunkPipeline<TInput, TOutput, TSub, TSubOut, U, CInput, COutput, R, RSub>(
this.pipeline,
fanOutFn,
subPipeline,
fanInFn
),
this.builderContext
)
}
/**
* Group items by key and process the groups concurrently, optionally capped
* by `maxConcurrency`. Results are returned unordered as each group completes.
*
* The callback composes how a group's queued chunk is processed from group
* chunk stages (`pipeChunk`, `sequentially`), keeping within-group ordering
* visible at the call site.
*
* @param options.maxConcurrency - Cap on how many groups process at once. Omitted means unbounded.
*/
concurrentlyPerGroup<TKey, U, ROut extends string = never>(
groupingFn: GroupingFunction<TOutput, TKey>,
callback: (
group: GroupProcessingBuilder<TInput, TOutput, CInput, COutput, R, D>
) => GroupProcessingBuilder<TInput, U, CInput, COutput, ROut, D>,
options?: { maxConcurrency?: number }
): ChunkPipelineBuilder<TInput, U, CInput, COutput, R | ROut, D> {
const group = callback(
new GroupProcessingBuilder(
<U2, R2 extends string>(processGroupChunk: GroupChunkProcessor<TOutput, U2, COutput, R2>) =>
new ConcurrentlyGroupingChunkPipeline(
groupingFn,
processGroupChunk,
this.pipeline,
options?.maxConcurrency
),
this.builderContext
)
)
return new ChunkPipelineBuilder(
group.buildGroupedPipeline((items) => Promise.resolve(items)),
this.builderContext
)
}
handleSideEffects(
promiseScheduler: PromiseScheduler,
options: { await: boolean }
): ChunkPipelineBuilder<TInput, TOutput, CInput, COutput, R, D> {
return new ChunkPipelineBuilder(
new SideEffectHandlingPipeline(this.pipeline, promiseScheduler, options),
this.builderContext
)
}
messageAware<TOut, COut = COutput, ROut extends string = never>(
this: ChunkPipelineBuilder<
TInput,
TOutput,
CInput & { message: Message },
COutput & { message: Message },
R,
D
>,
callback: (
builder: ChunkPipelineBuilder<
TInput,
TOutput,
CInput & { message: Message },
COutput & { message: Message },
R,
D
>
) => ChunkPipelineBuilder<TInput, TOut, CInput & { message: Message }, COut & { message: Message }, ROut, D>
): MessageAwareChunkPipelineBuilder<
TInput,
TOut,
CInput & { message: Message },
COut & { message: Message },
ROut,
D
> {
const builtPipeline = callback(this)
return new MessageAwareChunkPipelineBuilder(builtPipeline.build(), builtPipeline.builderContext)
}
teamAware<TOut, COut = COutput, ROut extends string = never>(
this: ChunkPipelineBuilder<TInput, TOutput, CInput & TeamIdContext, COutput & TeamIdContext, R, D>,
callback: (
builder: ChunkPipelineBuilder<TInput, TOutput, CInput & TeamIdContext, COutput & TeamIdContext, R, D>
) => ChunkPipelineBuilder<TInput, TOut, CInput & TeamIdContext, COut & TeamIdContext, ROut, D>
): TeamAwareChunkPipelineBuilder<TInput, TOut, CInput & TeamIdContext, COut & TeamIdContext, ROut, D> {
const builtPipeline = callback(this)
return new TeamAwareChunkPipelineBuilder(builtPipeline.build(), builtPipeline.builderContext)
}
build(): ChunkPipeline<TInput, TOutput, CInput, COutput, R> {
return this.pipeline
}
}
export class MessageAwareChunkPipelineBuilder<
TInput,
TOutput,
CInput extends { message: Message },
COutput extends { message: Message } = CInput,
R extends string = never,
D = unknown,
> {
constructor(
protected pipeline: ChunkPipeline<TInput, TOutput, CInput, COutput, R>,
protected builderContext?: PipelineBuilderContext<D>
) {}
handleResults<RConfig extends string = never>(
config: PipelineConfig<R | RConfig>
): ResultHandledChunkPipelineBuilder<TInput, TOutput, CInput, COutput, R, D> {
return new ResultHandledChunkPipelineBuilder(
new ResultHandlingPipeline(this.pipeline, config),
this.builderContext
)
}
}
/**
* Builder returned after handleResults(). Only allows handleSideEffects() to be called,
* enforcing that side effects must be handled before building.
*/
export class ResultHandledChunkPipelineBuilder<
TInput,
TOutput,
CInput extends { message: Message },
COutput extends { message: Message } = CInput,
R extends string = never,
D = unknown,
> {
constructor(
protected pipeline: ChunkPipeline<TInput, TOutput, CInput, COutput, R>,
protected builderContext?: PipelineBuilderContext<D>
) {}
handleSideEffects(
promiseScheduler: PromiseScheduler,
options: { await: boolean }
): ChunkPipelineBuilder<TInput, TOutput, CInput, COutput, R, D> {
return new ChunkPipelineBuilder(
new SideEffectHandlingPipeline(this.pipeline, promiseScheduler, options),
this.builderContext
)
}
}
export class TeamAwareChunkPipelineBuilder<
TInput,
TOutput,
CInput extends TeamIdContext,
COutput extends TeamIdContext,
R extends string = never,
D = unknown,
> extends ChunkPipelineBuilder<TInput, TOutput, CInput, COutput, R, D> {
constructor(
pipeline: ChunkPipeline<TInput, TOutput, CInput, COutput, R>,
builderContext?: PipelineBuilderContext<D>
) {
super(pipeline, builderContext)
}
handleIngestionWarnings(
outputs: IngestionOutputs<IngestionWarningsOutput>
): ChunkPipelineBuilder<TInput, TOutput, CInput, COutput, R, D> {
return new ChunkPipelineBuilder(
new IngestionWarningHandlingChunkPipeline(outputs, this.pipeline),
this.builderContext
)
}
}