:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import {
ClientMetrics,
HighLevelProducer,
LibrdKafkaError,
MessageHeader,
MessageValue,
Metadata,
NumberNullUndefined,
ProducerGlobalConfig,
MessageKey as RdKafkaMessageKey,
} from 'node-rdkafka'
import { hostname } from 'os'
import { Counter, Histogram, Summary } from 'prom-client'
import { instrumentFn } from '~/common/tracing/tracing-utils'
import { DependencyUnavailableError, MessageSizeTooLarge } from '../utils/db/error'
import { logger } from '../utils/logger'
import { KafkaConfigTarget, getKafkaConfigFromEnv } from './config'
import { ProducerStatsTracker } from './kafka-producer-metrics'
/** This class is a wrapper around the rdkafka producer, and does very little.
*
* The big difference between this and the original is that we return a promise from
* queueMessage, which will only resolve once we get an ack that the message has
* been persisted to Kafka. So we should get stronger guarantees on processing.
*
* TODO: refactor Kafka producer usage to use rdkafka directly.
*/
export type MessageKey = Exclude<RdKafkaMessageKey, undefined>
export type MessageWithoutTopic = {
value: string | Buffer | null
key?: MessageKey
headers?: Record<string, string>
topic?: never
}
export type TopicMessage = {
topic: string
messages: MessageWithoutTopic[]
}
export type ProducerMessage = {
value: MessageValue
key: MessageKey
topic: string
headers?: Record<string, string>
/** Explicit target partition. When omitted, librdkafka's default partitioner picks one. */
partition?: number
}
export interface KafkaProducer {
produce(message: ProducerMessage): Promise<void>
queueMessages(topicMessages: TopicMessage | TopicMessage[]): Promise<void>
disconnect(): Promise<void>
checkConnection(timeoutMs?: number): Promise<void>
checkTopicExists(topic: string, timeoutMs?: number): Promise<void>
}
export class KafkaProducerWrapper implements KafkaProducer {
/** Kafka producer used for syncing Postgres and ClickHouse person data. */
private producer: HighLevelProducer
private static readonly PRODUCER_DEFAULTS: ProducerGlobalConfig = {
'client.id': hostname(),
'metadata.broker.list': 'kafka:9092',
'linger.ms': 20,
log_level: 4, // WARN as the default
'batch.size': 8 * 1024 * 1024,
'queue.buffering.max.messages': 100_000,
'compression.codec': 'snappy',
'enable.idempotence': true,
'metadata.max.age.ms': 30000, // Refresh metadata every 30s
'retry.backoff.ms': 500, // Backoff between retry attempts
'socket.timeout.ms': 30000, // Timeout for socket operations
'max.in.flight.requests.per.connection': 5, // Required for idempotence ordering
}
/** Emit librdkafka stats every 30s so ProducerStatsTracker can export them as Prom metrics. */
private static readonly STATS_INTERVAL_MS = 30000
static async create(kafkaClientRack: string | undefined, mode: KafkaConfigTarget = 'PRODUCER') {
// NOTE: In addition to some defaults we allow overriding any setting via env vars.
// This makes it much easier to react to issues without needing code changes
return KafkaProducerWrapper.createWithConfig(kafkaClientRack, getKafkaConfigFromEnv(mode))
}
/** Create a producer with a pre-built rdkafka config (merged over defaults). */
static async createWithConfig(
kafkaClientRack: string | undefined,
config: ProducerGlobalConfig,
name?: string
): Promise<KafkaProducerWrapper> {
const producerConfig: ProducerGlobalConfig = {
...KafkaProducerWrapper.PRODUCER_DEFAULTS,
'client.rack': kafkaClientRack,
// Only emit librdkafka stats when there's a named wrapper to consume them — avoids
// the per-interval serialization cost on unnamed producers that have no listener.
...(name ? { 'statistics.interval.ms': KafkaProducerWrapper.STATS_INTERVAL_MS } : {}),
...config,
dr_cb: true,
}
logger.info('📝', 'librdkafka producer config', { config: producerConfig })
const producer = new HighLevelProducer(producerConfig)
producer.on('event.log', function (log) {
logger.info('📝', 'librdkafka log', { log: log })
})
producer.on('event.error', function (err) {
logger.error('📝', 'librdkafka error', { log: err })
})
await new Promise((resolve, reject) =>
producer.connect(undefined, (error, data) => {
if (error) {
logger.error('⚠️', 'connect_error', { error: error })
reject(error)
} else {
logger.info('📝', 'librdkafka producer connected', { error, brokers: data?.brokers })
resolve(data)
}
})
)
return new KafkaProducerWrapper(producer, name)
}
/** Optional human-readable name (e.g. 'DEFAULT', 'WARPSTREAM') used in metrics labels. */
public readonly name?: string
constructor(producer: HighLevelProducer, name?: string) {
this.producer = producer
this.name = name
if (name) {
const statsTracker = new ProducerStatsTracker(name)
producer.on('event.stats', (event) => statsTracker.track(event.message))
}
}
async produce({ value, key, topic, headers, partition }: ProducerMessage): Promise<void> {
const labels: Record<string, string> = { topic_name: topic }
if (this.name) {
labels.producer_name = this.name
}
try {
const produceTimer = ingestEventKafkaProduceLatency.labels(labels).startTimer()
const produceLatencyTimer = kafkaProducerProduceLatencySeconds.labels(labels).startTimer()
kafkaProducerMessagesQueuedCounter.labels(labels).inc()
if (value) {
kafkaProducerMessageSizeBytes.labels(labels).observe(value.length)
}
logger.debug('📤', 'Producing message', { topic: topic })
// NOTE: The MessageHeader type is super weird. Essentially you are passing in a record and it expects a string key and a string or buffer value.
const kafkaHeaders: MessageHeader[] =
Object.entries(headers ?? {}).map(([key, value]) => ({
[key]: value,
})) ?? []
const result = await instrumentFn(
{
key: 'kafka_produce_callback',
timeoutMs: 10_000,
sendException: false,
timeoutMessage: `Kafka produce callback timeout for topic "${topic}"`,
span: false,
},
() =>
new Promise((resolve, reject) => {
this.producer.produce(
topic,
partition ?? null,
value,
key,
Date.now(),
kafkaHeaders,
(error: any, offset: NumberNullUndefined) => {
return error ? reject(error) : resolve(offset)
}
)
})
)
kafkaProducerMessagesWrittenCounter.labels(labels).inc()
logger.debug('📤', 'Produced message', { topic: topic, offset: result })
produceTimer()
produceLatencyTimer()
} catch (error) {
const errorCode = (error as LibrdKafkaError)?.code
const failureLabels = {
...labels,
error_code: typeof errorCode === 'number' ? String(errorCode) : 'unknown',
}
kafkaProducerMessagesFailedCounter.labels(failureLabels).inc()
logger.error('⚠️', 'kafka_produce_error', {
error: typeof error?.message === 'string' ? error.message : JSON.stringify(error),
topic: topic,
error_code: errorCode,
})
if ((error as LibrdKafkaError).isRetriable) {
// If we get a retriable error, bubble that up so that the
// caller can retry.
throw new DependencyUnavailableError(error.message, 'Kafka', error)
} else if ((error as LibrdKafkaError).code === 10) {
throw new MessageSizeTooLarge(error.message, error)
}
throw error
}
}
/**
* Currently this produces messages in parallel.
* If ordering is required then you should use the `produce` method instead in an awaited loop.
*/
async queueMessages(topicMessages: TopicMessage | TopicMessage[]): Promise<void> {
topicMessages = Array.isArray(topicMessages) ? topicMessages : [topicMessages]
await Promise.all(
topicMessages.map((record) => {
return Promise.all(
record.messages.map((message) =>
this.produce({
topic: record.topic,
key: message.key ? Buffer.from(message.key) : null,
value: message.value ? Buffer.from(message.value) : null,
headers: message.headers,
})
)
)
})
)
}
public async flush() {
logger.debug('📤', 'flushing_producer')
return await new Promise((resolve, reject) =>
this.producer.flush(10000, (error) => {
logger.debug('📤', 'flushed_producer')
if (error) {
reject(error)
} else {
resolve(null)
}
})
)
}
public async disconnect(): Promise<void> {
logger.info('🔌', 'Disconnecting producer. Flushing...')
await this.flush()
this.producer.removeAllListeners('event.stats')
logger.info('🔌', 'Disconnecting producer. Disconnecting...')
await new Promise<ClientMetrics>((resolve, reject) =>
this.producer.disconnect((error: any, data: ClientMetrics) => {
logger.info('🔌', 'Disconnected producer')
if (error) {
reject(error)
} else {
resolve(data)
}
})
)
}
/** Check that the producer can reach the broker. */
public async checkConnection(timeoutMs = 10000): Promise<void> {
await new Promise<Metadata>((resolve, reject) =>
this.producer.getMetadata({ timeout: timeoutMs }, (error, data) => {
if (error) {
reject(error)
} else {
resolve(data)
}
})
)
}
/** Check that a topic exists on the broker. */
public async checkTopicExists(topic: string, timeoutMs = 10000): Promise<void> {
const metadata = await new Promise<Metadata>((resolve, reject) =>
this.producer.getMetadata({ topic, timeout: timeoutMs }, (error, data) => {
if (error) {
reject(error)
} else {
resolve(data)
}
})
)
const topicMeta = metadata.topics.find((t) => t.name === topic)
if (!topicMeta) {
throw new Error(`Topic "${topic}" not found in broker metadata`)
}
if (topicMeta.partitions.length === 0) {
throw new Error(`Topic "${topic}" has no partitions`)
}
}
}
export const kafkaProducerMessagesQueuedCounter = new Counter({
name: 'kafka_producer_messages_queued_total',
help: 'Count of messages queued to the Kafka producer, by destination topic.',
labelNames: ['topic_name', 'producer_name'],
})
export const kafkaProducerMessagesWrittenCounter = new Counter({
name: 'kafka_producer_messages_written_total',
help: 'Count of messages written to Kafka, by destination topic.',
labelNames: ['topic_name', 'producer_name'],
})
export const kafkaProducerMessagesFailedCounter = new Counter({
name: 'kafka_producer_messages_failed_total',
help: 'Count of write failures by the Kafka producer, by destination topic and librdkafka error code.',
labelNames: ['topic_name', 'producer_name', 'error_code'],
})
export const ingestEventKafkaProduceLatency = new Summary({
name: 'ingest_event_kafka_produce_latency',
help: 'Wait time for individual Kafka produces',
labelNames: ['topic_name', 'producer_name'],
percentiles: [0.5, 0.9, 0.95, 0.99],
})
export const kafkaProducerProduceLatencySeconds = new Histogram({
name: 'kafka_producer_produce_latency_seconds',
help: 'Latency of individual Kafka produces, from queue time to delivery ack.',
labelNames: ['topic_name', 'producer_name'],
buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 45, 60],
})
export const kafkaProducerMessageSizeBytes = new Histogram({
name: 'kafka_producer_message_size_bytes',
help: 'Size distribution of messages produced to Kafka, measured at queue time.',
labelNames: ['topic_name', 'producer_name'],
// Targeting typical event sizes (~1 KB) up to max request size (~1 MB).
buckets: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304],
})