Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.8 KB
Raw Blame
import { Message } from 'node-rdkafka'
import { Gauge } from 'prom-client'

import { TophogOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { instrumentFn } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { IngestionConsumerConfig } from '~/ingestion/config'
import { BatchResult, FeedResult } from '~/ingestion/framework/batching-pipeline'
import { createKafkaDebugContext, createOkContext } from '~/ingestion/framework/helpers'
import { OkResultWithContext } from '~/ingestion/framework/pipeline.interface'
import { TopHog, TopHogComponent } from '~/ingestion/framework/tophog'
import { HealthCheckResult, PluginServerService } from '~/types'

import { Scope, extend } from './scopes'
import { KafkaConsumerComponent, KafkaConsumerInterface } from './utils/kafka-consumer'
import { PromiseScheduler } from './utils/promise-scheduler'

type MessageInput = { message: Message }
type MessageContext = { message: Message }

export interface IngestionBatchingPipeline {
    feed(
        elements: OkResultWithContext<MessageInput, MessageContext>[],
        batchContext: Record<never, never>
    ): Promise<FeedResult>
    next(): Promise<BatchResult<unknown> | null>
}

export interface PipelineFactoryContext<S extends Record<string, object>> {
    container: S
}

export type PipelineFactory<S extends Record<string, object>> = (
    ctx: PipelineFactoryContext<S>
) => IngestionBatchingPipeline

/**
 * Constraint on a scope's container: must expose a `promiseScheduler`.
 * The pipeline schedules its side effects there; the common consumer
 * pulls it from the container to await them after each batch. The
 * scheduler is owned by the user-provided scope so its lifetime spans
 * the consumer's lifetime.
 */
export type ContainerWithPromiseScheduler = Record<string, object> & { promiseScheduler: PromiseScheduler }

export type CommonIngestionConsumerConfig = Pick<
    IngestionConsumerConfig,
    | 'INGESTION_CONSUMER_GROUP_ID'
    | 'INGESTION_CONSUMER_CONSUME_TOPIC'
    | 'INGESTION_PIPELINE'
    | 'INGESTION_LANE'
    | 'KAFKA_BATCH_START_LOGGING_ENABLED'
>

const latestOffsetTimestampGauge = new Gauge({
    name: 'common_ingestion_latest_processed_timestamp_ms',
    help: 'Timestamp of the latest offset that has been committed.',
    labelNames: ['topic', 'partition', 'groupId'],
    aggregator: 'max',
})

/**
 * Setup-side of a common ingestion consumer. Constructing it extends the
 * user-provided scope with a `kafkaConsumer` entry; `start()` brings up
 * the parent scope, builds the pipeline (sync), connects the Kafka
 * consumer, and returns a live `CommonIngestionConsumer` that owns the
 * started handle.
 */
export class CommonIngestionConsumerScope<
    S extends ContainerWithPromiseScheduler & { outputs: IngestionOutputs<TophogOutput> },
> {
    private readonly innerScope: Scope<S & { topHog: TopHog } & { kafkaConsumer: KafkaConsumerInterface }>

    constructor(
        private readonly name: string,
        config: CommonIngestionConsumerConfig,
        scope: Scope<S>,
        pipelineFactory: PipelineFactory<S & { topHog: TopHog }>
    ) {
        const consumerName = this.name
        // Every consumer gets a topHog registry scoped to its own outputs and
        // pipeline/lane labels. It lives one scope above the consumer so its
        // started value is in the container when the pipeline factory runs —
        // and so its flush loop starts before, and stops after, the consumer.
        const scopeWithTopHog = extend(scope, `${consumerName}-tophog`, (container, builder) =>
            builder.add(
                'topHog',
                new TopHogComponent({
                    outputs: container.outputs,
                    pipeline: config.INGESTION_PIPELINE ?? 'unknown',
                    lane: config.INGESTION_LANE ?? 'unknown',
                })
            )
        )
        this.innerScope = extend(scopeWithTopHog, `${consumerName}-consumer`, (container, builder) => {
            const pipeline = pipelineFactory({ container })
            const handler = new KafkaBatchHandler(config, consumerName, pipeline, container.promiseScheduler)
            return builder.add(
                'kafkaConsumer',
                new KafkaConsumerComponent(
                    config.INGESTION_CONSUMER_GROUP_ID,
                    config.INGESTION_CONSUMER_CONSUME_TOPIC,
                    (messages) => handler.handle(messages)
                )
            )
        })
    }

    async start(): Promise<{ consumer: CommonIngestionConsumer; stop: () => Promise<void> }> {
        const started = await this.innerScope.start()
        const consumer = new CommonIngestionConsumer(this.name, started.container.kafkaConsumer)
        return { consumer, stop: started.stop }
    }
}

/**
 * Live, started common ingestion consumer. Exposes only what surrounding
 * code actually needs from the started scope: the Kafka consumer (for
 * health) and the per-process healthcheck. Lifetime is owned by the
 * scope handle returned alongside it from `CommonIngestionConsumerScope.start()`.
 */
export class CommonIngestionConsumer {
    constructor(
        readonly name: string,
        private readonly kafkaConsumer: KafkaConsumerInterface
    ) {}

    isHealthy(): HealthCheckResult {
        return this.kafkaConsumer.isHealthy()
    }
}

/**
 * Builds a `PluginServerService` descriptor for a started common
 * ingestion consumer. The scope handle's `stop` is supplied explicitly
 * so lifecycle ownership stays with the scope, not the consumer.
 */
export function ingestionConsumerService(
    consumer: CommonIngestionConsumer,
    stop: () => Promise<void>
): PluginServerService {
    return {
        id: consumer.name,
        onShutdown: stop,
        healthcheck: () => consumer.isHealthy(),
    }
}

/**
 * Per-consumer wrapper that owns the deps a Kafka batch handler needs
 * (config, name, pipeline, promise scheduler) and exposes a single
 * `handle(messages)` method matching what `KafkaConsumerComponent` expects.
 * Internal — only constructed inside `CommonIngestionConsumerScope`.
 */
class KafkaBatchHandler {
    constructor(
        private readonly config: CommonIngestionConsumerConfig,
        private readonly name: string,
        private readonly pipeline: IngestionBatchingPipeline,
        private readonly promiseScheduler: PromiseScheduler
    ) {}

    async handle(messages: Message[]): Promise<{ backgroundTask?: Promise<unknown> }> {
        if (this.config.KAFKA_BATCH_START_LOGGING_ENABLED) {
            this.logBatchStart(messages)
        }

        const batch = messages.map((message) =>
            createOkContext({ message }, { message, debugContext: createKafkaDebugContext(message) })
        )

        const feedResult = await this.pipeline.feed(batch, {})
        if (!feedResult.ok) {
            throw new Error(`Pipeline rejected batch: ${feedResult.reason}`)
        }

        // The pipeline handles its own side effects (scheduling them on the
        // promise scheduler), so draining results is all that's left to do.
        let result = await this.pipeline.next()
        while (result !== null) {
            result = await this.pipeline.next()
        }

        for (const message of messages) {
            if (message.timestamp) {
                latestOffsetTimestampGauge
                    .labels({
                        partition: message.partition,
                        topic: message.topic,
                        groupId: this.config.INGESTION_CONSUMER_GROUP_ID,
                    })
                    .set(message.timestamp)
            }
        }

        return {
            backgroundTask: instrumentFn('commonIngestionConsumer.awaitScheduledWork', async () => {
                await this.promiseScheduler.waitForAll()
            }),
        }
    }

    private logBatchStart(messages: Message[]): void {
        const podName = process.env.HOSTNAME || 'unknown'
        const partitionEarliestMessages = new Map<number, Message>()
        const partitionBatchSizes = new Map<number, number>()

        messages.forEach((message) => {
            const existing = partitionEarliestMessages.get(message.partition)
            if (!existing || message.offset < existing.offset) {
                partitionEarliestMessages.set(message.partition, message)
            }
            partitionBatchSizes.set(message.partition, (partitionBatchSizes.get(message.partition) || 0) + 1)
        })

        const partitionData = Array.from(partitionEarliestMessages.entries()).map(([partition, message]) => ({
            partition,
            offset: message.offset,
            batchSize: partitionBatchSizes.get(partition) || 0,
        }))

        logger.info('📖', `KAFKA_BATCH_START: ${this.name}`, {
            pod: podName,
            totalMessages: messages.length,
            partitions: partitionData,
        })
    }
}