:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/**
* # Chapter 13: Pipeline Conventions
*
* This chapter documents the conventions and patterns used when writing
* pipeline steps and composing pipelines in the ingestion codebase.
*
* ## Overview
*
* - **Factory functions**: Steps are created via factory functions for DI and naming
* - **Type extension**: Steps progressively enrich data using `T & NewProps`
* - **Configuration injection**: Factory functions accept config parameters
* - **Void returns**: Terminal steps return `void` to signal end of processing
* - **Pipeline phases**: Preprocessing (pre/post team) vs processing separation
* - **Subpipelines**: Large pipelines broken into composable pieces
*
* ## Type Definition Conventions
*
* **Best practice**: Define input, output, and config types separately from
* function definitions. This improves readability and allows types to be
* reused across multiple steps.
*
* **Never use `any`**: The framework is designed to leverage TypeScript's type
* checker while minimizing boilerplate. If you find yourself needing `any`, it
* indicates a structural problem with the code. Use `unknown` when the type is
* genuinely unknown. This applies to tests as well - using `any` in tests can
* mask actual issues with the code being tested.
*
* **Omit redundant type annotations**: Types for inner function arguments and
* return types should be omitted - TypeScript infers these from the outer
* function's return type annotation. This reduces noise and keeps code readable.
*/
import { Message } from 'node-rdkafka'
import { DLQ_OUTPUT, INGESTION_WARNINGS_OUTPUT, IngestionWarningsOutput, OVERFLOW_OUTPUT } from '~/common/outputs'
import { parseJSON } from '~/common/utils/json-parse'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { newChunkPipelineBuilder, newPipelineBuilder } from '~/ingestion/framework/builders'
import { PipelineBuilder, StartPipelineBuilder } from '~/ingestion/framework/builders/pipeline-builders'
import { createOkContext } from '~/ingestion/framework/helpers'
import { PipelineConfig } from '~/ingestion/framework/result-handling-pipeline'
import { dlq, isOkResult, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { createTestMessage } from '~/tests/helpers/kafka-message'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
import { createTestTeam } from '~/tests/helpers/team'
import { Team } from '~/types'
describe('Factory Functions', () => {
/**
* Steps are created via factory functions that return the actual step.
* This pattern enables:
* - Dependency injection (pass services, config)
* - Named functions for better stack traces
* - Closure over dependencies
*/
it('steps are created via factory functions', async () => {
interface Input {
value: string
}
interface Output {
value: string
processed: boolean
}
// Factory function returns a named step function
function createProcessStep(): ProcessingStep<Input, Output> {
return function processStep(input) {
return Promise.resolve(ok({ value: input.value.toUpperCase(), processed: true }))
}
}
const pipeline = newPipelineBuilder<Input>().pipe(createProcessStep()).build()
const result = await pipeline.process(createOkContext({ value: 'hello' }, {}))
expect(isOkResult(result.result) && result.result.value).toEqual({
value: 'HELLO',
processed: true,
})
})
/**
* Whole pipelines are also created via factory functions. This is essential
* because chunk pipelines are stateful (they maintain internal buffers via
* feed/next) and should only have one caller. Creating pipelines via factory
* functions ensures each consumer gets its own instance.
*/
it('pipelines are created via factory functions to ensure single caller', async () => {
interface Input {
id: number
}
interface Output {
id: number
processed: boolean
}
function createProcessStep(): ProcessingStep<Input, Output> {
return function processStep(input) {
return Promise.resolve(ok({ id: input.id, processed: true }))
}
}
// Factory function creates a new pipeline instance each time
function createMyPipeline() {
return newPipelineBuilder<Input>().pipe(createProcessStep()).build()
}
// Each caller gets their own pipeline instance
const pipeline1 = createMyPipeline()
const pipeline2 = createMyPipeline()
// They are separate instances
expect(pipeline1).not.toBe(pipeline2)
// Each can be used independently
const result1 = await pipeline1.process(createOkContext({ id: 1 }, {}))
const result2 = await pipeline2.process(createOkContext({ id: 2 }, {}))
expect(isOkResult(result1.result) && result1.result.value.id).toBe(1)
expect(isOkResult(result2.result) && result2.result.value.id).toBe(2)
})
})
describe('Configuration Injection', () => {
/**
* Factory functions accept configuration parameters for dependency injection.
* This allows steps to be configured at pipeline construction time.
*/
it('factory functions accept configuration parameters', async () => {
interface Input {
value: number
}
interface Output {
value: number
multiplied: number
}
interface MultiplyStepConfig {
multiplier: number
}
// Factory accepts config and closes over it
function createMultiplyStep(config: MultiplyStepConfig): ProcessingStep<Input, Output> {
return function multiplyStep(input) {
return Promise.resolve(
ok({
value: input.value,
multiplied: input.value * config.multiplier,
})
)
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createMultiplyStep({ multiplier: 3 }))
.build()
const result = await pipeline.process(createOkContext({ value: 10 }, {}))
expect(isOkResult(result.result) && result.result.value.multiplied).toBe(30)
})
/**
* Complex steps may need multiple dependencies injected.
*/
it('factory functions can inject multiple dependencies', async () => {
interface Input {
userId: string
}
interface Output {
userId: string
userData: { name: string }
auditLogged: boolean
}
// Mock services
const mockUserService = {
getUser: (id: string) => Promise.resolve({ name: `User ${id}` }),
}
const mockAuditService = {
log: (_msg: string) => Promise.resolve(),
}
interface EnrichStepDependencies {
userService: typeof mockUserService
auditService: typeof mockAuditService
}
function createEnrichStep(deps: EnrichStepDependencies): ProcessingStep<Input, Output> {
return async function enrichStep(input) {
const userData = await deps.userService.getUser(input.userId)
const auditPromise = deps.auditService.log(`Enriched user ${input.userId}`)
return ok({ userId: input.userId, userData, auditLogged: true }, [auditPromise])
}
}
const pipeline = newPipelineBuilder<Input>()
.pipe(createEnrichStep({ userService: mockUserService, auditService: mockAuditService }))
.build()
const result = await pipeline.process(createOkContext({ userId: '123' }, {}))
expect(isOkResult(result.result) && result.result.value.userData.name).toBe('User 123')
})
})
describe('Type Extension', () => {
/**
* Steps use generic constraints to declare required input properties,
* then extend the input type with new properties using `T & NewProps`.
* This allows data to accumulate through the pipeline.
*/
it('steps extend input types with new properties', async () => {
// Step 1 only needs 'raw' property
interface ValidateInput {
raw: string
}
interface ValidateOutput {
isValid: boolean
}
// Step 2 needs 'raw' and 'isValid'
interface EnrichInput {
raw: string
isValid: boolean
}
interface EnrichOutput {
enrichedAt: number
}
// Generic constraint: T must have ValidateInput properties
// Output: T plus ValidateOutput (preserves all input properties)
function createValidateStep<T extends ValidateInput>(): ProcessingStep<T, T & ValidateOutput> {
return function validateStep(input) {
return Promise.resolve(ok({ ...input, isValid: input.raw.length > 0 }))
}
}
function createEnrichStep<T extends EnrichInput>(): ProcessingStep<T, T & EnrichOutput> {
return function enrichStep(input) {
return Promise.resolve(ok({ ...input, enrichedAt: Date.now() }))
}
}
const pipeline = newPipelineBuilder<{ raw: string }>()
.pipe(createValidateStep())
.pipe(createEnrichStep())
.build()
const result = await pipeline.process(createOkContext({ raw: 'test data' }, {}))
// Final result has all accumulated properties
expect(isOkResult(result.result)).toBe(true)
if (isOkResult(result.result)) {
expect(result.result.value.raw).toBe('test data')
expect(result.result.value.isValid).toBe(true)
expect(result.result.value.enrichedAt).toBeGreaterThan(0)
}
})
/**
* Some steps transform input into a completely different shape.
* These steps don't use generic extension - they define explicit input/output types.
*/
it('terminal transforms produce different output shapes', async () => {
interface ProcessedEvent {
eventId: string
isValid: boolean
timestamp: number
}
interface EventSummary {
summary: string
}
// No generic - produces completely different shape
function createSummarizeStep(): ProcessingStep<ProcessedEvent, EventSummary> {
return function summarizeStep(input) {
return Promise.resolve(
ok({
summary: `Event ${input.eventId}: valid=${input.isValid} at ${input.timestamp}`,
})
)
}
}
const pipeline = newPipelineBuilder<ProcessedEvent>().pipe(createSummarizeStep()).build()
const result = await pipeline.process(createOkContext({ eventId: 'evt-1', isValid: true, timestamp: 1000 }, {}))
expect(isOkResult(result.result) && result.result.value.summary).toBe('Event evt-1: valid=true at 1000')
})
})
describe('Void Returns', () => {
/**
* Terminal steps that don't pass data forward return `void`.
* They use `ok(undefined, [...sideEffects])` to signal completion
* while still attaching side effects.
*/
it('terminal steps return void with side effects', async () => {
const emittedEvents: string[] = []
interface EventToEmit {
eventId: string
data: string
}
// Terminal step - returns void, no further processing
function createEmitStep(): ProcessingStep<EventToEmit, void> {
return function emitStep(input) {
const emitPromise = Promise.resolve().then(() => {
emittedEvents.push(input.eventId)
})
// Return ok(undefined) with side effect
return Promise.resolve(ok(undefined, [emitPromise]))
}
}
const pipeline = newPipelineBuilder<EventToEmit>().pipe(createEmitStep()).build()
const result = await pipeline.process(createOkContext({ eventId: 'evt-1', data: 'test' }, {}))
// Result is OK with undefined value
expect(isOkResult(result.result)).toBe(true)
if (isOkResult(result.result)) {
expect(result.result.value).toBeUndefined()
}
// Side effect was captured
await Promise.all(result.context.sideEffects)
expect(emittedEvents).toContain('evt-1')
})
/**
* Simple terminal steps may not need side effects at all.
*/
it('simple terminal steps return ok(undefined)', async () => {
interface Input {
shouldSkip: boolean
}
// Skip step - just ends processing without doing anything
function createSkipStep(): ProcessingStep<Input, void> {
return function skipStep(_input) {
return Promise.resolve(ok(undefined))
}
}
const pipeline = newPipelineBuilder<Input>().pipe(createSkipStep()).build()
const result = await pipeline.process(createOkContext({ shouldSkip: true }, {}))
expect(isOkResult(result.result)).toBe(true)
})
})
describe('Pipeline Phases', () => {
/**
* Large pipelines are organized into phases, each as a separate subpipeline:
* 1. Pre-team preprocessing: parsing, validation before team lookup
* 2. Post-team preprocessing: team-specific validation, enrichment
* 3. Processing: core business logic, event creation, emission
*
* This separation allows:
* - Early rejection of invalid events (before expensive team lookup)
* - Team-specific logic isolated from parsing
* - Clear boundaries for testing and maintenance
* - Independent testing of each phase
*/
it('pipelines are organized into preprocessing and processing phases', async () => {
const phaseLog: string[] = []
const promiseScheduler = new PromiseScheduler()
interface RawInput {
message: Message
}
interface PreTeamOutput {
teamId: number
eventName: string
team: Team
}
interface PostTeamOutput {
validated: boolean
}
interface ProcessingOutput {
result: string
}
// Steps for each phase (single-item steps used inside sequentially)
function createParseStep<T extends RawInput>(): ProcessingStep<T, T & PreTeamOutput> {
return function parseStep(item) {
phaseLog.push('pre-team: parse')
try {
const parsed = parseJSON(item.message.value?.toString() ?? '{}')
const team = createTestTeam({ id: parsed.teamId })
return Promise.resolve(ok({ ...item, teamId: parsed.teamId, eventName: parsed.eventName, team }))
} catch (e) {
return Promise.resolve(dlq('Failed to parse message', e as Error))
}
}
}
function createValidateStep<T extends PreTeamOutput>(): ProcessingStep<T, T & PostTeamOutput> {
return function validateStep(item) {
phaseLog.push('post-team: validate')
return Promise.resolve(ok({ ...item, validated: true }))
}
}
function createProcessStep<T extends PreTeamOutput & PostTeamOutput>(): ProcessingStep<
T,
T & ProcessingOutput
> {
return function processStep(item) {
phaseLog.push('processing: process')
return Promise.resolve(ok({ ...item, result: `${item.eventName} for Team ${item.teamId}` }))
}
}
// Subpipelines take a StartPipelineBuilder and return a PipelineBuilder
function createPreTeamPreprocessingSubpipeline<T extends RawInput, C>(
builder: StartPipelineBuilder<T, C>
): PipelineBuilder<T, T & PreTeamOutput, C> {
return builder.pipe(createParseStep<T>())
}
function createPostTeamPreprocessingSubpipeline<T extends PreTeamOutput, C>(
builder: StartPipelineBuilder<T, C>
): PipelineBuilder<T, T & PostTeamOutput, C> {
return builder.pipe(createValidateStep<T>())
}
function createProcessingSubpipeline<T extends PreTeamOutput & PostTeamOutput, C>(
builder: StartPipelineBuilder<T, C>
): PipelineBuilder<T, T & ProcessingOutput, C> {
return builder.pipe(createProcessStep<T>())
}
const pipelineConfig: PipelineConfig = {
outputs: createMockIngestionOutputs<
typeof DLQ_OUTPUT | typeof OVERFLOW_OUTPUT | typeof INGESTION_WARNINGS_OUTPUT
>(),
promiseScheduler,
}
// Compose subpipelines like joined-ingestion-pipeline:
// messageAware → concurrently(preTeam) → filterMap(addTeam,
// teamAware(concurrently(postTeam) → concurrentlyPerGroup(processing))
// → handleIngestionWarnings)
// → handleResults → handleSideEffects
function createPipeline() {
return newChunkPipelineBuilder<RawInput, { message: Message }>()
.messageAware((b) =>
b
// Pre-team preprocessing: parse and resolve team (concurrent)
.concurrently((b) => createPreTeamPreprocessingSubpipeline(b))
// Add team to context, then process team-aware steps
.filterMap(
(element) => ({
result: element.result,
context: { ...element.context, team: element.result.value.team },
}),
(b) =>
b
.teamAware((b) =>
b
// Post-team preprocessing: validate (concurrent)
.concurrently((b) => createPostTeamPreprocessingSubpipeline(b))
// Processing: group by team; groups run concurrently while the
// explicit `sequentially` step processes items within a group in order
.concurrentlyPerGroup(
(item) => item.teamId,
(group) => group.sequentially((b) => createProcessingSubpipeline(b))
)
)
.handleIngestionWarnings(createMockIngestionOutputs<IngestionWarningsOutput>())
)
)
.handleResults(pipelineConfig)
.handleSideEffects(promiseScheduler, { await: true })
.build()
}
const pipeline = createPipeline()
const message = createTestMessage({ value: Buffer.from(JSON.stringify({ teamId: 42, eventName: 'pageview' })) })
const batch = [createOkContext({ message } as RawInput, { message })]
pipeline.feed(batch)
const results = await pipeline.next()
expect(phaseLog).toEqual(['pre-team: parse', 'post-team: validate', 'processing: process'])
expect(results).toHaveLength(1)
if (isOkResult(results![0].result)) {
expect(results![0].result.value.result).toBe('pageview for Team 42')
}
})
})