Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

3.0 KB
Raw Blame
import { Assignment, Message, PartitionMetadata, TopicPartitionOffset } from 'node-rdkafka'

import { HealthCheckResult } from '~/types'

import { defaultConfig } from '../../config/config'
import { logger } from '../../utils/logger'
import { KafkaConsumer, KafkaConsumerConfig, RdKafkaConsumerConfig } from './consumer-v1'
import { KafkaConsumerV2 } from './consumer-v2'

// Re-exports for back-compat with code that imports header parsers from `kafka/consumer`.
// These utilities are not really part of the consumer class and could move out later.
export { parseEventHeaders, parseKafkaHeaders } from './consumer-v1'
export type { KafkaConsumerConfig, RdKafkaConsumerConfig } from './consumer-v1'

/**
 * Overrides `auto.offset.reset` so a fresh consumer group starts at the tip of the topic instead
 * of replaying the retained backlog. `auto.offset.reset` is a topic-level librdkafka property, so
 * the global config type has no slot for it; both implementations mirror the value into the
 * explicit topic config, which is the form librdkafka honors (see node-rdkafka #984).
 */
export const START_AT_LATEST = { ['auto.offset.reset' as keyof RdKafkaConsumerConfig]: 'latest' as never }

/**
 * The shared surface that both KafkaConsumer (v1) and KafkaConsumerV2 expose to call sites.
 * Keep this small — every method here pins the migration. Adding a method here forces both
 * implementations to support it.
 */
export interface KafkaConsumerInterface {
    connect(
        eachBatch: (messages: Message[]) => Promise<{ backgroundTask?: Promise<unknown> } | void>,
        onPartitionsRevoked?: (revokedPartitions: Assignment[]) => Promise<void>
    ): Promise<void>
    disconnect(): Promise<void>
    isHealthy(): HealthCheckResult
    offsetsStore(offsets: TopicPartitionOffset[]): void
    /** Broker-side `[low, high]` offsets for a partition. `high` is the next offset to be produced. */
    queryWatermarkOffsets(topic: string, partition: number, timeout?: number): Promise<[number, number]>
    /** The group's committed offsets for the currently assigned partitions. */
    committedOffsets(timeout?: number): Promise<TopicPartitionOffset[]>
    /** Broker-reported partitions of a topic, whether or not this consumer is assigned them. */
    getPartitionsForTopic(topic: string): Promise<PartitionMetadata[]>
}

/**
 * Single entry-point for instantiating a Kafka consumer. v1/v2 selection is driven by the
 * `CONSUMER_USE_V2` env var (boolean, applies to the entire service). Call sites should
 * never `new KafkaConsumer(...)` directly during the rollout — go through this factory.
 */
export function createKafkaConsumer(
    config: KafkaConsumerConfig,
    rdKafkaConfig: RdKafkaConsumerConfig = {}
): KafkaConsumerInterface {
    if (defaultConfig.CONSUMER_USE_V2) {
        logger.info('🔁', 'kafka_consumer_factory_using_v2', { groupId: config.groupId, topic: config.topic })
        return new KafkaConsumerV2(config, rdKafkaConfig)
    }
    return new KafkaConsumer(config, rdKafkaConfig)
}