Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.2 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { DlqOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { ingestionPipelineResultCounter } from '~/ingestion/common/metrics'
import {
    logDroppedMessage,
    produceMessageToDLQ,
    redirectMessageToOutput,
} from '~/ingestion/framework/result-handling-helpers'

import { ChunkPipeline, ChunkPipelineResultWithContext, OkResultWithContext } from './chunk-pipeline.interface'
import {
    PipelineResult,
    PipelineResultRedirect,
    PipelineResultType,
    isDlqResult,
    isDropResult,
    isOkResult,
    isRedirectResult,
} from './results'

export type PipelineConfig<R extends string = never> = {
    outputs: IngestionOutputs<DlqOutput | R>
    promiseScheduler: PromiseScheduler
    /** Reject the side effect when the DLQ produce fails. Set it only when the consumer stores no offsets after a rejected side effect. */
    rejectOnDlqFailure?: boolean
}

/**
 * Unified result handling pipeline that wraps any chunk-processing pipeline and handles
 * non-success results (DLQ, DROP, REDIRECT, TIMEOUT) by adding side effects.
 *
 * The redirect type R flows through — redirect results are kept in the output
 * with the redirect side effect attached.
 *
 * A TIMEOUT produces nothing, because its redelivery produces it.
 */
export class ResultHandlingPipeline<
    TInput,
    TOutput,
    CInput extends { message: Message },
    COutput extends { message: Message } = CInput,
    R extends string = never,
> implements ChunkPipeline<TInput, TOutput, CInput, COutput, R>
{
    constructor(
        private pipeline: ChunkPipeline<TInput, TOutput, CInput, COutput, R>,
        private config: PipelineConfig<R>
    ) {}

    feed(elements: OkResultWithContext<TInput, CInput>[]): void {
        this.pipeline.feed(elements)
    }

    async next(): Promise<ChunkPipelineResultWithContext<TOutput, COutput, R> | null> {
        const results = await this.pipeline.next()

        if (results === null) {
            return null
        }

        const processedResults: ChunkPipelineResultWithContext<TOutput, COutput, R> = []

        for (const resultWithContext of results) {
            const stepName = resultWithContext.context.lastStep || 'unknown'
            const { result: resultType, details } = resultDetails(resultWithContext.result)
            ingestionPipelineResultCounter.labels({ result: resultType, last_step_name: stepName, details }).inc()

            if (isOkResult(resultWithContext.result)) {
                processedResults.push(resultWithContext)
            } else {
                const result = resultWithContext.result
                const originalMessage = resultWithContext.context.message
                const sideEffects = this.handleNonSuccessResult(result, originalMessage, stepName)

                processedResults.push({
                    result,
                    context: {
                        ...resultWithContext.context,
                        sideEffects: [...resultWithContext.context.sideEffects, ...sideEffects],
                    },
                })
            }
        }

        return processedResults
    }

    private handleNonSuccessResult(
        result: PipelineResult<TOutput, R>,
        originalMessage: Message,
        stepName: string
    ): Promise<unknown>[] {
        const sideEffects: Promise<unknown>[] = []

        if (isDlqResult(result)) {
            const error = result.error || new Error(result.reason)
            const dlqPromise = this.config.rejectOnDlqFailure
                ? produceMessageToDLQ(this.config.outputs, originalMessage, error, stepName, { rethrowOnFailure: true })
                : produceMessageToDLQ(this.config.outputs, originalMessage, error, stepName)
            sideEffects.push(dlqPromise)
        } else if (isDropResult(result)) {
            logDroppedMessage(originalMessage, result.reason, stepName)
        } else if (isRedirectResult(result)) {
            const redirectPromise = redirectMessageToOutput(
                this.config.outputs,
                result.output,
                this.config.promiseScheduler,
                originalMessage,
                stepName,
                result.preserveKey ?? true,
                result.awaitAck ?? true
            )
            sideEffects.push(redirectPromise)
        }

        return sideEffects
    }
}

function redirectDetails<R extends string>(result: PipelineResultRedirect<R>): string {
    const preserveKey = result.preserveKey ?? true
    return `${result.output}(preserve_key=${preserveKey})`
}

function resultDetails<T, R extends string>(result: PipelineResult<T, R>): { result: string; details: string } {
    switch (result.type) {
        case PipelineResultType.OK:
            return { result: 'ok', details: '' }
        case PipelineResultType.DROP:
            return { result: 'drop', details: result.reason }
        case PipelineResultType.DLQ:
            return { result: 'dlq', details: result.reason }
        case PipelineResultType.REDIRECT:
            return { result: 'redirect', details: redirectDetails(result) }
        case PipelineResultType.TIMEOUT:
            return { result: 'timeout', details: result.reason }
    }
}