Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.4 KB
Raw Blame
import {
    ClientMetrics,
    HighLevelProducer,
    LibrdKafkaError,
    MessageHeader,
    MessageValue,
    Metadata,
    NumberNullUndefined,
    ProducerGlobalConfig,
    MessageKey as RdKafkaMessageKey,
} from 'node-rdkafka'
import { hostname } from 'os'
import { Counter, Histogram, Summary } from 'prom-client'

import { instrumentFn } from '~/common/tracing/tracing-utils'

import { DependencyUnavailableError, MessageSizeTooLarge } from '../utils/db/error'
import { logger } from '../utils/logger'
import { KafkaConfigTarget, getKafkaConfigFromEnv } from './config'
import { ProducerStatsTracker } from './kafka-producer-metrics'

/** This class is a wrapper around the rdkafka producer, and does very little.
 *
 * The big difference between this and the original is that we return a promise from
 * queueMessage, which will only resolve once we get an ack that the message has
 * been persisted to Kafka. So we should get stronger guarantees on processing.
 *
 * TODO: refactor Kafka producer usage to use rdkafka directly.
 */

export type MessageKey = Exclude<RdKafkaMessageKey, undefined>

export type MessageWithoutTopic = {
    value: string | Buffer | null
    key?: MessageKey
    headers?: Record<string, string>
    topic?: never
}

export type TopicMessage = {
    topic: string
    messages: MessageWithoutTopic[]
}

export type ProducerMessage = {
    value: MessageValue
    key: MessageKey
    topic: string
    headers?: Record<string, string>
    /** Explicit target partition. When omitted, librdkafka's default partitioner picks one. */
    partition?: number
}

export interface KafkaProducer {
    produce(message: ProducerMessage): Promise<void>
    queueMessages(topicMessages: TopicMessage | TopicMessage[]): Promise<void>
    disconnect(): Promise<void>
    checkConnection(timeoutMs?: number): Promise<void>
    checkTopicExists(topic: string, timeoutMs?: number): Promise<void>
}

export class KafkaProducerWrapper implements KafkaProducer {
    /** Kafka producer used for syncing Postgres and ClickHouse person data. */
    private producer: HighLevelProducer

    private static readonly PRODUCER_DEFAULTS: ProducerGlobalConfig = {
        'client.id': hostname(),
        'metadata.broker.list': 'kafka:9092',
        'linger.ms': 20,
        log_level: 4, // WARN as the default
        'batch.size': 8 * 1024 * 1024,
        'queue.buffering.max.messages': 100_000,
        'compression.codec': 'snappy',
        'enable.idempotence': true,
        'metadata.max.age.ms': 30000, // Refresh metadata every 30s
        'retry.backoff.ms': 500, // Backoff between retry attempts
        'socket.timeout.ms': 30000, // Timeout for socket operations
        'max.in.flight.requests.per.connection': 5, // Required for idempotence ordering
    }

    /** Emit librdkafka stats every 30s so ProducerStatsTracker can export them as Prom metrics. */
    private static readonly STATS_INTERVAL_MS = 30000

    static async create(kafkaClientRack: string | undefined, mode: KafkaConfigTarget = 'PRODUCER') {
        // NOTE: In addition to some defaults we allow overriding any setting via env vars.
        // This makes it much easier to react to issues without needing code changes
        return KafkaProducerWrapper.createWithConfig(kafkaClientRack, getKafkaConfigFromEnv(mode))
    }

    /** Create a producer with a pre-built rdkafka config (merged over defaults). */
    static async createWithConfig(
        kafkaClientRack: string | undefined,
        config: ProducerGlobalConfig,
        name?: string
    ): Promise<KafkaProducerWrapper> {
        const producerConfig: ProducerGlobalConfig = {
            ...KafkaProducerWrapper.PRODUCER_DEFAULTS,
            'client.rack': kafkaClientRack,
            // Only emit librdkafka stats when there's a named wrapper to consume them — avoids
            // the per-interval serialization cost on unnamed producers that have no listener.
            ...(name ? { 'statistics.interval.ms': KafkaProducerWrapper.STATS_INTERVAL_MS } : {}),
            ...config,
            dr_cb: true,
        }

        logger.info('📝', 'librdkafka producer config', { config: producerConfig })

        const producer = new HighLevelProducer(producerConfig)

        producer.on('event.log', function (log) {
            logger.info('📝', 'librdkafka log', { log: log })
        })

        producer.on('event.error', function (err) {
            logger.error('📝', 'librdkafka error', { log: err })
        })

        await new Promise((resolve, reject) =>
            producer.connect(undefined, (error, data) => {
                if (error) {
                    logger.error('⚠️', 'connect_error', { error: error })
                    reject(error)
                } else {
                    logger.info('📝', 'librdkafka producer connected', { error, brokers: data?.brokers })
                    resolve(data)
                }
            })
        )

        return new KafkaProducerWrapper(producer, name)
    }

    /** Optional human-readable name (e.g. 'DEFAULT', 'WARPSTREAM') used in metrics labels. */
    public readonly name?: string

    constructor(producer: HighLevelProducer, name?: string) {
        this.producer = producer
        this.name = name

        if (name) {
            const statsTracker = new ProducerStatsTracker(name)
            producer.on('event.stats', (event) => statsTracker.track(event.message))
        }
    }

    async produce({ value, key, topic, headers, partition }: ProducerMessage): Promise<void> {
        const labels: Record<string, string> = { topic_name: topic }
        if (this.name) {
            labels.producer_name = this.name
        }
        try {
            const produceTimer = ingestEventKafkaProduceLatency.labels(labels).startTimer()
            const produceLatencyTimer = kafkaProducerProduceLatencySeconds.labels(labels).startTimer()
            kafkaProducerMessagesQueuedCounter.labels(labels).inc()
            if (value) {
                kafkaProducerMessageSizeBytes.labels(labels).observe(value.length)
            }
            logger.debug('📤', 'Producing message', { topic: topic })

            // NOTE: The MessageHeader type is super weird. Essentially you are passing in a record and it expects a string key and a string or buffer value.
            const kafkaHeaders: MessageHeader[] =
                Object.entries(headers ?? {}).map(([key, value]) => ({
                    [key]: value,
                })) ?? []

            const result = await instrumentFn(
                {
                    key: 'kafka_produce_callback',
                    timeoutMs: 10_000,
                    sendException: false,
                    timeoutMessage: `Kafka produce callback timeout for topic "${topic}"`,
                    span: false,
                },
                () =>
                    new Promise((resolve, reject) => {
                        this.producer.produce(
                            topic,
                            partition ?? null,
                            value,
                            key,
                            Date.now(),
                            kafkaHeaders,
                            (error: any, offset: NumberNullUndefined) => {
                                return error ? reject(error) : resolve(offset)
                            }
                        )
                    })
            )

            kafkaProducerMessagesWrittenCounter.labels(labels).inc()
            logger.debug('📤', 'Produced message', { topic: topic, offset: result })
            produceTimer()
            produceLatencyTimer()
        } catch (error) {
            const errorCode = (error as LibrdKafkaError)?.code
            const failureLabels = {
                ...labels,
                error_code: typeof errorCode === 'number' ? String(errorCode) : 'unknown',
            }
            kafkaProducerMessagesFailedCounter.labels(failureLabels).inc()
            logger.error('⚠️', 'kafka_produce_error', {
                error: typeof error?.message === 'string' ? error.message : JSON.stringify(error),
                topic: topic,
                error_code: errorCode,
            })

            if ((error as LibrdKafkaError).isRetriable) {
                // If we get a retriable error, bubble that up so that the
                // caller can retry.
                throw new DependencyUnavailableError(error.message, 'Kafka', error)
            } else if ((error as LibrdKafkaError).code === 10) {
                throw new MessageSizeTooLarge(error.message, error)
            }

            throw error
        }
    }

    /**
     * Currently this produces messages in parallel.
     * If ordering is required then you should use the `produce` method instead in an awaited loop.
     */
    async queueMessages(topicMessages: TopicMessage | TopicMessage[]): Promise<void> {
        topicMessages = Array.isArray(topicMessages) ? topicMessages : [topicMessages]

        await Promise.all(
            topicMessages.map((record) => {
                return Promise.all(
                    record.messages.map((message) =>
                        this.produce({
                            topic: record.topic,
                            key: message.key ? Buffer.from(message.key) : null,
                            value: message.value ? Buffer.from(message.value) : null,
                            headers: message.headers,
                        })
                    )
                )
            })
        )
    }

    public async flush() {
        logger.debug('📤', 'flushing_producer')

        return await new Promise((resolve, reject) =>
            this.producer.flush(10000, (error) => {
                logger.debug('📤', 'flushed_producer')
                if (error) {
                    reject(error)
                } else {
                    resolve(null)
                }
            })
        )
    }

    public async disconnect(): Promise<void> {
        logger.info('🔌', 'Disconnecting producer. Flushing...')
        await this.flush()

        this.producer.removeAllListeners('event.stats')

        logger.info('🔌', 'Disconnecting producer. Disconnecting...')
        await new Promise<ClientMetrics>((resolve, reject) =>
            this.producer.disconnect((error: any, data: ClientMetrics) => {
                logger.info('🔌', 'Disconnected producer')
                if (error) {
                    reject(error)
                } else {
                    resolve(data)
                }
            })
        )
    }

    /** Check that the producer can reach the broker. */
    public async checkConnection(timeoutMs = 10000): Promise<void> {
        await new Promise<Metadata>((resolve, reject) =>
            this.producer.getMetadata({ timeout: timeoutMs }, (error, data) => {
                if (error) {
                    reject(error)
                } else {
                    resolve(data)
                }
            })
        )
    }

    /** Check that a topic exists on the broker. */
    public async checkTopicExists(topic: string, timeoutMs = 10000): Promise<void> {
        const metadata = await new Promise<Metadata>((resolve, reject) =>
            this.producer.getMetadata({ topic, timeout: timeoutMs }, (error, data) => {
                if (error) {
                    reject(error)
                } else {
                    resolve(data)
                }
            })
        )

        const topicMeta = metadata.topics.find((t) => t.name === topic)
        if (!topicMeta) {
            throw new Error(`Topic "${topic}" not found in broker metadata`)
        }
        if (topicMeta.partitions.length === 0) {
            throw new Error(`Topic "${topic}" has no partitions`)
        }
    }
}

export const kafkaProducerMessagesQueuedCounter = new Counter({
    name: 'kafka_producer_messages_queued_total',
    help: 'Count of messages queued to the Kafka producer, by destination topic.',
    labelNames: ['topic_name', 'producer_name'],
})

export const kafkaProducerMessagesWrittenCounter = new Counter({
    name: 'kafka_producer_messages_written_total',
    help: 'Count of messages written to Kafka, by destination topic.',
    labelNames: ['topic_name', 'producer_name'],
})

export const kafkaProducerMessagesFailedCounter = new Counter({
    name: 'kafka_producer_messages_failed_total',
    help: 'Count of write failures by the Kafka producer, by destination topic and librdkafka error code.',
    labelNames: ['topic_name', 'producer_name', 'error_code'],
})

export const ingestEventKafkaProduceLatency = new Summary({
    name: 'ingest_event_kafka_produce_latency',
    help: 'Wait time for individual Kafka produces',
    labelNames: ['topic_name', 'producer_name'],
    percentiles: [0.5, 0.9, 0.95, 0.99],
})

export const kafkaProducerProduceLatencySeconds = new Histogram({
    name: 'kafka_producer_produce_latency_seconds',
    help: 'Latency of individual Kafka produces, from queue time to delivery ack.',
    labelNames: ['topic_name', 'producer_name'],
    buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 45, 60],
})

export const kafkaProducerMessageSizeBytes = new Histogram({
    name: 'kafka_producer_message_size_bytes',
    help: 'Size distribution of messages produced to Kafka, measured at queue time.',
    labelNames: ['topic_name', 'producer_name'],
    // Targeting typical event sizes (~1 KB) up to max request size (~1 MB).
    buckets: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304],
})