Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.7 KB
Raw Blame
import { Message, MessageHeader } from 'node-rdkafka'

import { ChunkPipelineBuilder, newChunkPipelineBuilder } from './builders'
import { ChunkPipelineUnwrapper } from './chunk-pipeline-unwrapper'
import { ChunkPipeline } from './chunk-pipeline.interface'
import {
    DebugContextOf,
    OkResultWithContext,
    PipelineBuilderContext,
    PipelineDebugContext,
    PipelineWarning,
} from './pipeline.interface'
import { PipelineResult, ok } from './results'
import { StartPipeline } from './start-pipeline'

/** Loggable origin of a Kafka message, stored on the context at feed time. */
export type KafkaDebugContext = {
    topic?: string
    partition: number
    offset: number
    headers?: Record<string, string>
}

// The headers a responder needs to write an event restriction for a poison
// message. distinct_id and session_id are customer-controlled and stay out.
const LOGGED_HEADERS = new Set(['token', 'event', 'uuid'])

export type DefaultContext = { message: Message; debugContext?: KafkaDebugContext }

export function createKafkaDebugContext(message: Message): KafkaDebugContext {
    const { topic, partition, offset, headers } = message
    return {
        topic,
        partition,
        offset,
        // Built for every message, read only when a step crashes: render the
        // headers on read so the happy path does no string work.
        get headers() {
            return headers === undefined ? undefined : formatKafkaHeaders(headers)
        },
    }
}

function formatKafkaHeaders(headers: MessageHeader[]): Record<string, string> {
    const result: Record<string, string> = {}
    for (const header of headers) {
        for (const key of Object.keys(header)) {
            if (LOGGED_HEADERS.has(key)) {
                result[key] = header[key].toString()
            }
        }
    }
    return result
}

/**
 * Compact a list of offsets into a human-readable ranges string, e.g.
 * [1, 2, 3, 7, 10, 9] -> "1-3,7,9-10". Duplicates are collapsed.
 */
export function compactOffsetRanges(offsets: number[]): string {
    const sorted = [...new Set(offsets)].sort((a, b) => a - b)
    const ranges: string[] = []
    let start: number | null = null
    let end = 0
    for (const offset of sorted) {
        if (start === null) {
            start = end = offset
        } else if (offset === end + 1) {
            end = offset
        } else {
            ranges.push(start === end ? `${start}` : `${start}-${end}`)
            start = end = offset
        }
    }
    if (start !== null) {
        ranges.push(start === end ? `${start}` : `${start}-${end}`)
    }
    return ranges.join(',')
}

/**
 * Folds a crashed chunk's Kafka debug contexts into one compact summary: one
 * offsets-ranges string per topic-partition, so the log stays a single
 * readable line instead of one object per message. Duplicate offsets (e.g.
 * fan-out sub-elements sharing a parent) collapse in the range compaction.
 */
export function aggregateKafkaDebugContexts(debugContexts: KafkaDebugContext[]): PipelineDebugContext {
    const offsetsByPartition = new Map<string, number[]>()
    for (const debugContext of debugContexts) {
        const key = `${debugContext.topic}[${debugContext.partition}]`
        const offsets = offsetsByPartition.get(key)
        if (offsets) {
            offsets.push(debugContext.offset)
        } else {
            offsetsByPartition.set(key, [debugContext.offset])
        }
    }
    return {
        count: debugContexts.length,
        offsets: Array.from(offsetsByPartition, ([key, offsets]) => `${key}:${compactOffsetRanges(offsets)}`),
    }
}

/**
 * Helper function to create a new processing pipeline for single items
 */
export function createNewPipeline<T = { message: Message }, C = DefaultContext>(): StartPipeline<T, C> {
    return new StartPipeline<T, C>()
}

/**
 * Helper function to create a new chunk processing pipeline starting with a root pipeline
 */
export function createNewChunkPipeline<T = { message: Message }, C = DefaultContext>(
    builderContext?: PipelineBuilderContext<DebugContextOf<C>>
): ChunkPipelineBuilder<T, T, C, C, never, DebugContextOf<C>> {
    return newChunkPipelineBuilder<T, C>(builderContext)
}

/**
 * Helper function to create a batch of ResultWithContext from Kafka messages or objects with a message property
 */
export function createBatch<T extends DefaultContext>(items: T[]) {
    return items.map((item) =>
        createOkContext(item, { message: item.message, debugContext: createKafkaDebugContext(item.message) })
    )
}

/**
 * Base context properties that are always present in pipeline context
 */
export type BasePipelineContext = {
    lastStep?: string
    debugContext?: unknown
    sideEffects?: Promise<unknown>[]
    warnings?: PipelineWarning[]
}

/**
 * Result type for createContext that represents the actual shape of the returned context
 */
export type CreateContextResult<T, PartialContext, R extends string = never> = {
    result: PipelineResult<T, R>
    context: {
        lastStep: string | undefined
        sideEffects: Promise<unknown>[]
        warnings: PipelineWarning[]
    } & PartialContext
}

/**
 * Helper function to create a PipelineResultWithContext from a result and partial context
 */
export function createContext<
    T,
    PartialContext extends Record<string, unknown> = Record<string, never>,
    R extends string = never,
>(
    result: PipelineResult<T, R>,
    ...args: PartialContext extends Record<string, never>
        ? [partialContext?: PartialContext & BasePipelineContext]
        : [partialContext: PartialContext & BasePipelineContext]
): CreateContextResult<T, PartialContext, R> {
    const partialContext = args[0] || ({} as PartialContext & BasePipelineContext)
    const { lastStep, sideEffects, warnings, ...rest } = partialContext
    return {
        result,
        context: {
            lastStep: lastStep,
            sideEffects: sideEffects || [],
            warnings: warnings || [],
            ...rest,
        } as CreateContextResult<T, PartialContext>['context'],
    }
}

/**
 * Create an OK result with context, suitable for feeding into a pipeline via feed().
 */
export function createOkContext<T, C extends Record<string, unknown> = Record<string, never>>(
    value: T,
    partialContext: C & BasePipelineContext
): OkResultWithContext<T, C> {
    return {
        result: ok(value),
        context: {
            lastStep: undefined,
            sideEffects: [],
            warnings: [],
            ...partialContext,
        },
    }
}

/**
 * Helper function to create a chunk pipeline unwrapper
 */
export function createUnwrapper<TInput, TOutput, C, R extends string = never>(
    chunkPipeline: ChunkPipeline<TInput, TOutput, C, C, R>
): ChunkPipelineUnwrapper<TInput, TOutput, C, R> {
    return new ChunkPipelineUnwrapper(chunkPipeline)
}