:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { DlqOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { ingestionPipelineResultCounter } from '~/ingestion/common/metrics'
import {
logDroppedMessage,
produceMessageToDLQ,
redirectMessageToOutput,
} from '~/ingestion/framework/result-handling-helpers'
import { ChunkPipeline, ChunkPipelineResultWithContext, OkResultWithContext } from './chunk-pipeline.interface'
import {
PipelineResult,
PipelineResultRedirect,
PipelineResultType,
isDlqResult,
isDropResult,
isOkResult,
isRedirectResult,
} from './results'
export type PipelineConfig<R extends string = never> = {
outputs: IngestionOutputs<DlqOutput | R>
promiseScheduler: PromiseScheduler
/** Reject the side effect when the DLQ produce fails. Set it only when the consumer stores no offsets after a rejected side effect. */
rejectOnDlqFailure?: boolean
}
/**
* Unified result handling pipeline that wraps any chunk-processing pipeline and handles
* non-success results (DLQ, DROP, REDIRECT, TIMEOUT) by adding side effects.
*
* The redirect type R flows through — redirect results are kept in the output
* with the redirect side effect attached.
*
* A TIMEOUT produces nothing, because its redelivery produces it.
*/
export class ResultHandlingPipeline<
TInput,
TOutput,
CInput extends { message: Message },
COutput extends { message: Message } = CInput,
R extends string = never,
> implements ChunkPipeline<TInput, TOutput, CInput, COutput, R>
{
constructor(
private pipeline: ChunkPipeline<TInput, TOutput, CInput, COutput, R>,
private config: PipelineConfig<R>
) {}
feed(elements: OkResultWithContext<TInput, CInput>[]): void {
this.pipeline.feed(elements)
}
async next(): Promise<ChunkPipelineResultWithContext<TOutput, COutput, R> | null> {
const results = await this.pipeline.next()
if (results === null) {
return null
}
const processedResults: ChunkPipelineResultWithContext<TOutput, COutput, R> = []
for (const resultWithContext of results) {
const stepName = resultWithContext.context.lastStep || 'unknown'
const { result: resultType, details } = resultDetails(resultWithContext.result)
ingestionPipelineResultCounter.labels({ result: resultType, last_step_name: stepName, details }).inc()
if (isOkResult(resultWithContext.result)) {
processedResults.push(resultWithContext)
} else {
const result = resultWithContext.result
const originalMessage = resultWithContext.context.message
const sideEffects = this.handleNonSuccessResult(result, originalMessage, stepName)
processedResults.push({
result,
context: {
...resultWithContext.context,
sideEffects: [...resultWithContext.context.sideEffects, ...sideEffects],
},
})
}
}
return processedResults
}
private handleNonSuccessResult(
result: PipelineResult<TOutput, R>,
originalMessage: Message,
stepName: string
): Promise<unknown>[] {
const sideEffects: Promise<unknown>[] = []
if (isDlqResult(result)) {
const error = result.error || new Error(result.reason)
const dlqPromise = this.config.rejectOnDlqFailure
? produceMessageToDLQ(this.config.outputs, originalMessage, error, stepName, { rethrowOnFailure: true })
: produceMessageToDLQ(this.config.outputs, originalMessage, error, stepName)
sideEffects.push(dlqPromise)
} else if (isDropResult(result)) {
logDroppedMessage(originalMessage, result.reason, stepName)
} else if (isRedirectResult(result)) {
const redirectPromise = redirectMessageToOutput(
this.config.outputs,
result.output,
this.config.promiseScheduler,
originalMessage,
stepName,
result.preserveKey ?? true,
result.awaitAck ?? true
)
sideEffects.push(redirectPromise)
}
return sideEffects
}
}
function redirectDetails<R extends string>(result: PipelineResultRedirect<R>): string {
const preserveKey = result.preserveKey ?? true
return `${result.output}(preserve_key=${preserveKey})`
}
function resultDetails<T, R extends string>(result: PipelineResult<T, R>): { result: string; details: string } {
switch (result.type) {
case PipelineResultType.OK:
return { result: 'ok', details: '' }
case PipelineResultType.DROP:
return { result: 'drop', details: result.reason }
case PipelineResultType.DLQ:
return { result: 'dlq', details: result.reason }
case PipelineResultType.REDIRECT:
return { result: 'redirect', details: redirectDetails(result) }
case PipelineResultType.TIMEOUT:
return { result: 'timeout', details: result.reason }
}
}