:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/**
* # Chapter 1: Pipeline Fundamentals and Builders
*
* A pipeline is a linear sequence of processing steps that transform data.
* Each step receives input from the previous step, performs some work, and
* produces output for the next step.
*
* Pipelines are created using the builder pattern - this is the idiomatic
* and recommended way to compose pipelines in this framework.
*
* ## Key Concepts
*
* - **Single-item pipelines** process one item at a time using `newPipelineBuilder()`
* - **Chunk pipelines** process multiple items efficiently using `newChunkPipelineBuilder()`
* - **Steps** are functions that return a `Promise<PipelineResult<T>>`
* - **Results** can be OK (success), DLQ (error), DROP (discard), or REDIRECT (route elsewhere)
*
* ## Step Definition Pattern
*
* Steps follow a factory pattern:
* ```typescript
* function createMyStep(): ProcessingStep<Input, Output> {
* return function myStep(input) {
* return Promise.resolve(ok(transformedValue))
* }
* }
* ```
*/
import { newChunkPipelineBuilder, newPipelineBuilder } from '~/ingestion/framework/builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { PipelineResult, dlq, drop, isOkResult, ok, redirect } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
/**
* Type for chunk processing steps - takes an array of values and returns
* an array of results (must have same length).
*/
type ChunkProcessingStep<T, U> = (values: T[]) => Promise<PipelineResult<U>[]>
describe('Defining Steps', () => {
/**
* Steps are defined using the factory pattern: a function that returns
* the actual step function. This pattern enables:
*
* 1. **Dependency injection** - Pass config, clients, or other dependencies
* to the factory, which captures them in a closure
* 2. **Named functions** - The inner function has a name that appears in
* stack traces and the `lastStep` context field
* 3. **Type safety** - The factory return type `ProcessingStep<In, Out>`
* ensures the step signature is correct
*/
it('steps use the factory pattern for dependency injection and naming', async () => {
// A simple step with no dependencies
function createUppercaseStep(): ProcessingStep<string, string> {
return function uppercaseStep(input) {
return Promise.resolve(ok(input.toUpperCase()))
}
}
// A step that captures configuration via closure
interface MultiplierConfig {
factor: number
}
function createMultiplyStep(config: MultiplierConfig): ProcessingStep<number, number> {
return function multiplyStep(input) {
return Promise.resolve(ok(input * config.factor))
}
}
// Use steps in pipelines
const stringPipeline = newPipelineBuilder<string>().pipe(createUppercaseStep()).build()
const numberPipeline = newPipelineBuilder<number>()
.pipe(createMultiplyStep({ factor: 3 }))
.build()
const stringResult = await stringPipeline.process(createOkContext('hello', {}))
expect(isOkResult(stringResult.result) && stringResult.result.value).toBe('HELLO')
const numberResult = await numberPipeline.process(createOkContext(5, {}))
expect(isOkResult(numberResult.result) && numberResult.result.value).toBe(15)
})
/**
* Steps can add properties to their output, and subsequent steps receive
* all accumulated properties. This enables progressive enrichment where
* each step adds its own data to the result.
*
* Each step defines only the input properties it needs using `T extends Input`.
* This allows the step to work with any object that has those properties,
* passing through all other properties unchanged. This pattern allows
* composition of steps without unnecessary coupling - steps don't need to
* know about properties they don't use.
*/
it('steps accumulate data through the pipeline', async () => {
// Each step defines only the properties it NEEDS to read (Input)
// and the properties it ADDS (Output extends Input with new properties)
interface ValidateStepInput {
eventId: string
}
interface ValidateStepOutput {
isValid: boolean
}
interface EnrichStepInput {
isValid: boolean
}
interface EnrichStepOutput {
timestamp: number
}
interface FinalizeStepInput {
eventId: string
isValid: boolean
timestamp: number
}
// Note: FinalizeStepOutput does NOT extend the input - this step produces
// a summary and drops the intermediate properties. Steps can transform
// data into completely different shapes.
interface FinalizeStepOutput {
summary: string
}
// Step uses T extends Input so it accepts any object with required properties
// and passes through all other properties via spread
function createValidateStep<T extends ValidateStepInput>(): ProcessingStep<T, T & ValidateStepOutput> {
return function validateStep(input) {
return Promise.resolve(ok({ ...input, isValid: true }))
}
}
function createEnrichStep<T extends EnrichStepInput>(): ProcessingStep<T, T & EnrichStepOutput> {
return function enrichStep(input) {
return Promise.resolve(ok({ ...input, timestamp: Date.now() }))
}
}
// This step does NOT use T extends - it reads all accumulated properties
// and produces a new output shape, demonstrating that steps can also
// drop properties and transform data completely.
function createFinalizeStep(): ProcessingStep<FinalizeStepInput, FinalizeStepOutput> {
return function finalizeStep(input) {
// This step has access to ALL accumulated properties from previous steps
// and produces a summary, dropping the intermediate properties
return Promise.resolve(
ok({
summary: `Event ${input.eventId} valid=${input.isValid} at ${input.timestamp}`,
})
)
}
}
const pipeline = newPipelineBuilder<{ eventId: string }>()
.pipe(createValidateStep())
.pipe(createEnrichStep())
.pipe(createFinalizeStep())
.build()
const result = await pipeline.process(createOkContext({ eventId: 'evt-123' }, {}))
expect(isOkResult(result.result)).toBe(true)
if (isOkResult(result.result)) {
// Final result has ONLY the summary - previous properties were dropped
expect(result.result.value.summary).toContain('evt-123')
expect(result.result.value.summary).toContain('valid=true')
}
})
})
describe('Pipeline Fundamentals', () => {
/**
* A step can return OK to indicate successful processing and pass data
* forward to the next step.
*/
it('a step can return OK to pass data forward', async () => {
interface ProcessDataResult {
processed: string
}
function createProcessDataStep(): ProcessingStep<string, ProcessDataResult> {
return function processDataStep(input) {
return Promise.resolve(ok({ processed: input.toUpperCase() }))
}
}
const pipeline = newPipelineBuilder<string>().pipe(createProcessDataStep()).build()
const result = await pipeline.process(createOkContext('hello', {}))
expect(isOkResult(result.result)).toBe(true)
if (isOkResult(result.result)) {
expect(result.result.value).toEqual({ processed: 'HELLO' })
}
})
/**
* A pipeline transforms input through a sequence of steps.
* Each step receives the output of the previous step and returns
* a Promise<PipelineResult>.
*/
it('a pipeline transforms input through a sequence of steps', async () => {
function createUppercaseStep(): ProcessingStep<string, string> {
return function uppercaseStep(input) {
return Promise.resolve(ok(input.toUpperCase()))
}
}
function createAddExclamationStep(): ProcessingStep<string, string> {
return function addExclamationStep(input) {
return Promise.resolve(ok(input + '!'))
}
}
const pipeline = newPipelineBuilder<string>()
.pipe(createUppercaseStep())
.pipe(createAddExclamationStep())
.build()
const result = await pipeline.process(createOkContext('hello', {}))
expect(isOkResult(result.result)).toBe(true)
if (isOkResult(result.result)) {
expect(result.result.value).toBe('HELLO!')
}
})
/**
* A step can return DLQ (Dead Letter Queue) to signal an error that should
* be handled by error handling infrastructure.
*/
it('a step can return DLQ to signal an error', async () => {
interface ValidateDataInput {
value: number
}
function createValidateDataStep(): ProcessingStep<ValidateDataInput, ValidateDataInput> {
return function validateDataStep(input) {
if (input.value < 0) {
return Promise.resolve(dlq('Invalid value: must be non-negative', new Error('Validation failed')))
}
return Promise.resolve(ok(input))
}
}
const pipeline = newPipelineBuilder<ValidateDataInput>().pipe(createValidateDataStep()).build()
const result = await pipeline.process(createOkContext({ value: -1 }, {}))
expect(result.result.type).toBe(1) // PipelineResultType.DLQ
})
/**
* A step can return DROP to silently discard an item. This is useful for
* filtering out items that should not be processed further but are not errors.
*/
it('a step can return DROP to silently discard an item', async () => {
interface Event {
type: string
}
function createFilterInternalStep(): ProcessingStep<Event, Event> {
return function filterInternalStep(event) {
if (event.type === 'internal') {
return Promise.resolve(drop('Internal events are not processed'))
}
return Promise.resolve(ok(event))
}
}
const pipeline = newPipelineBuilder<Event>().pipe(createFilterInternalStep()).build()
const result = await pipeline.process(createOkContext({ type: 'internal' }, {}))
expect(result.result.type).toBe(2) // PipelineResultType.DROP
})
/**
* A step can return REDIRECT to send an item to a named output
* (e.g., overflow) instead of continuing through the pipeline.
*/
it('a step can return REDIRECT to send an item elsewhere', async () => {
interface PriorityEvent {
priority: string
}
function createRouteByPriorityStep(): ProcessingStep<PriorityEvent, PriorityEvent, 'high_priority'> {
return function routeByPriorityStep(event) {
if (event.priority === 'high') {
const HIGH_PRIORITY_OUTPUT = 'high_priority' as const
return Promise.resolve(redirect('High priority event', HIGH_PRIORITY_OUTPUT))
}
return Promise.resolve(ok(event))
}
}
const pipeline = newPipelineBuilder<PriorityEvent>().pipe(createRouteByPriorityStep()).build()
const result = await pipeline.process(createOkContext({ priority: 'high' }, {}))
expect(result.result.type).toBe(3) // PipelineResultType.REDIRECT
})
/**
* Non-OK results (DLQ, DROP, REDIRECT, TIMEOUT) short-circuit the pipeline - they
* skip all remaining steps. OK results continue to propagate through
* subsequent steps.
*/
it('non-OK results short-circuit the pipeline, OK results propagate', async () => {
type Input = { action: 'ok' | 'dlq' | 'drop' | 'redirect' }
type Output = { action: string; processed: true }
function createDecisionStep(): ProcessingStep<Input, Input, 'other'> {
return function decisionStep(input) {
if (input.action === 'dlq') {
return Promise.resolve(dlq('Error condition'))
}
if (input.action === 'drop') {
return Promise.resolve(drop('Filtered out'))
}
if (input.action === 'redirect') {
const OTHER_OUTPUT = 'other' as const
return Promise.resolve(redirect('Redirecting', OTHER_OUTPUT))
}
return Promise.resolve(ok(input))
}
}
function createProcessStep(): ProcessingStep<Input, Output> {
return function processStep(input) {
// This step expects only 'ok' actions - if short-circuiting failed,
// we'd receive dlq/drop/redirect actions and this would be wrong
if (input.action !== 'ok') {
throw new Error(`Unexpected action: ${input.action}`)
}
return Promise.resolve(ok({ action: input.action, processed: true }))
}
}
const pipeline = newPipelineBuilder<Input>().pipe(createDecisionStep()).pipe(createProcessStep()).build()
// OK result propagates through all steps
const okResult = await pipeline.process(createOkContext({ action: 'ok' }, {}))
expect(isOkResult(okResult.result)).toBe(true)
expect(okResult.context.lastStep).toBe('processStep')
// DLQ short-circuits - processStep is skipped
const dlqResult = await pipeline.process(createOkContext({ action: 'dlq' }, {}))
expect(dlqResult.result.type).toBe(1) // DLQ
expect(dlqResult.context.lastStep).toBe('decisionStep')
// DROP short-circuits - processStep is skipped
const dropResult = await pipeline.process(createOkContext({ action: 'drop' }, {}))
expect(dropResult.result.type).toBe(2) // DROP
expect(dropResult.context.lastStep).toBe('decisionStep')
// REDIRECT short-circuits - processStep is skipped
const redirectResult = await pipeline.process(createOkContext({ action: 'redirect' }, {}))
expect(redirectResult.result.type).toBe(3) // REDIRECT
expect(redirectResult.context.lastStep).toBe('decisionStep')
})
})
describe('Chunk Pipelines', () => {
/**
* Chunk pipelines are created using `newChunkPipelineBuilder()`. The builder
* provides methods for adding chunk steps, concurrent processing, and more.
*/
it('chunk pipelines are created using newChunkPipelineBuilder()', async () => {
function createUppercaseChunkStep(): ChunkProcessingStep<string, string> {
return function uppercaseChunkStep(items) {
return Promise.resolve(items.map((s) => ok(s.toUpperCase())))
}
}
const pipeline = newChunkPipelineBuilder<string>().pipeChunk(createUppercaseChunkStep()).build()
const batch = ['a', 'b', 'c'].map((s) => createOkContext(s, {}))
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).not.toBeNull()
expect(results!.map((r) => (isOkResult(r.result) ? r.result.value : null))).toEqual(['A', 'B', 'C'])
})
/**
* The `pipeChunk` method adds a step that processes the entire chunk at once.
* The step receives an array of values and must return an array of results
* with the same length.
*/
it('pipeChunk processes all items in a single call', async () => {
const callCounts: number[] = []
function createChunkEnrichStep(): ChunkProcessingStep<number, number> {
return function chunkEnrichStep(items) {
callCounts.push(items.length)
return Promise.resolve(items.map((n) => ok(n * 10)))
}
}
const pipeline = newChunkPipelineBuilder<number>().pipeChunk(createChunkEnrichStep()).build()
const batch = [1, 2, 3, 4, 5].map((n) => createOkContext(n, {}))
pipeline.feed(batch)
await pipeline.next()
// Step was called once with all 5 items
expect(callCounts).toEqual([5])
})
/**
* Chunk pipelines use a feed/next interface. Call `feed()` to add items
* to the pipeline, then call `next()` to get processed results. `next()`
* returns null when all items have been processed.
*/
it('feed() accepts a batch of items and next() returns processed chunks', async () => {
function createPassthroughStep(): ChunkProcessingStep<string, string> {
return function passthroughStep(items) {
return Promise.resolve(items.map((s) => ok(s)))
}
}
const pipeline = newChunkPipelineBuilder<string>().pipeChunk(createPassthroughStep()).build()
const batch = ['x', 'y'].map((s) => createOkContext(s, {}))
pipeline.feed(batch)
const results1 = await pipeline.next()
expect(results1).not.toBeNull()
expect(results1!.length).toBe(2)
const results2 = await pipeline.next()
expect(results2).toBeNull() // No more items
})
})