Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

40.8 KB
Raw Blame
import {
    Assignment,
    CODES,
    ClientMetrics,
    ConsumerGlobalConfig,
    LibrdKafkaError,
    Message,
    MessageHeader,
    Metadata,
    PartitionMetadata,
    KafkaConsumer as RdKafkaConsumer,
    TopicPartitionOffset,
    WatermarkOffsets,
} from 'node-rdkafka'
import { hostname } from 'os'
import { Counter, Histogram } from 'prom-client'

import { instrumentFn } from '~/common/tracing/tracing-utils'
import { sanitizeString } from '~/common/utils/db/utils'
import { isTestEnv } from '~/common/utils/env-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { normalizeSessionId } from '~/common/utils/utils'
import {
    EventHeaders,
    HealthCheckResult,
    HealthCheckResultDegraded,
    HealthCheckResultError,
    HealthCheckResultOk,
    LogLevel,
} from '~/types'

import { defaultConfig } from '../../config/config'
import { logger } from '../../utils/logger'
import { captureException } from '../../utils/posthog'
import { retryIfRetriable } from '../../utils/retries'
import { promisifyCallback } from '../../utils/utils'
import { ensureTopicExists } from '../admin'
import { getKafkaConfigFromEnv, stripClassicProtocolConfig } from '../config'
import { parseBrokerStatistics, trackBrokerMetrics } from '../kafka-client-metrics'
import {
    consumedBatchBackgroundDuration,
    consumedBatchBackpressureDuration,
    consumedBatchDuration,
    consumerBatchUtilization as gaugeBatchUtilization,
    consumerBatchSize as histogramKafkaBatchSize,
    consumerBatchSizeKb as histogramKafkaBatchSizeKb,
    kafkaConsumerAssignment,
} from './metrics'

const DEFAULT_BATCH_TIMEOUT_MS = 500
const SLOW_BATCH_PROCESSING_LOG_THRESHOLD_MS = 10000
const MAX_HEALTH_HEARTBEAT_INTERVAL_MS = 60_000
const STATISTICS_INTERVAL_MS = 5000 // Emit internal metrics every 5 seconds

const kafkaHeaderStatusCounter = new Counter({
    name: 'kafka_header_status_total',
    help: 'Count of events by header name and presence status',
    labelNames: ['header', 'status'],
})

// v1-only metrics — v2 doesn't surface these. Kept here until v1 is deleted.
const histogramKafkaConsumeInterval = new Histogram({
    name: 'kafka_consume_interval_ms',
    help: 'Time elapsed between Kafka consume calls',
    labelNames: ['topic', 'groupId'],
    buckets: [0, 20, 100, 200, 500, 1000, 2500, 5000, 10000, 20000, 30000, 60000, Infinity],
})

export const findOffsetsToCommit = (messages: TopicPartitionOffset[]): TopicPartitionOffset[] => {
    // We only need to commit the highest offset for a batch of messages
    const messagesByTopicPartition = messages.reduce(
        (acc, message) => {
            if (!acc[message.topic]) {
                acc[message.topic] = {}
            }

            if (!acc[message.topic][message.partition]) {
                acc[message.topic][message.partition] = []
            }

            acc[message.topic][message.partition].push(message)

            return acc
        },
        {} as { [topic: string]: { [partition: number]: TopicPartitionOffset[] } }
    )

    // Then we find the highest offset for each topic partition
    const highestOffsets = Object.entries(messagesByTopicPartition).flatMap(([topic, partitions]) => {
        return Object.entries(partitions).map(([partition, messages]) => {
            const highestOffset = Math.max(...messages.map((message) => message.offset))

            return {
                topic,
                partition: parseInt(partition),
                // When committing to Kafka you commit the offset of the next message you want to consume
                offset: highestOffset + 1,
            }
        })
    })

    return highestOffsets
}

export type KafkaConsumerConfig = {
    groupId: string
    topic: string
    batchTimeoutMs?: number
    callEachBatchWhenEmpty?: boolean
    autoOffsetStore?: boolean
    autoCommit?: boolean
    enablePartitionEof?: boolean
    /**
     * Messages pulled per poll, overriding CONSUMER_BATCH_SIZE for this consumer only.
     *
     * For lanes whose per-message work is slow or unbounded, batch size is what keeps a batch inside
     * max.poll.interval.ms, so it belongs next to the code that depends on it rather than in a
     * deployment value that can drift away from the assumption without anything noticing.
     */
    fetchBatchSize?: number
}

export type RdKafkaConsumerConfig = Omit<
    ConsumerGlobalConfig,
    'group.id' | 'enable.auto.offset.store' | 'enable.auto.commit'
>

interface RebalanceCoordination {
    isRebalancing: boolean
    rebalanceTimeoutMs: number
    rebalanceStartTime: number
}

export class KafkaConsumer {
    private isStopping = false
    private lastHeartbeatTime = 0
    private rdKafkaConsumer: RdKafkaConsumer
    private consumerConfig: ConsumerGlobalConfig
    private fetchBatchSize: number
    private maxHealthHeartbeatIntervalMs: number
    private maxBackgroundTasks: number
    private consumerLoop: Promise<void> | undefined
    private backgroundTask: { promise: Promise<void>; createdAt: number }[]
    private podName: string
    private lastBackgroundTaskCompletionTime: number
    private consumerId: string
    // New health monitoring state
    private consumerLoopStallThresholdMs: number
    private lastConsumerLoopTime = 0
    private consumerState: string | undefined
    private lastStatsEmitTime = 0
    private rebalanceCoordination: RebalanceCoordination = {
        isRebalancing: false,
        rebalanceTimeoutMs: defaultConfig.CONSUMER_REBALANCE_TIMEOUT_MS,
        rebalanceStartTime: 0,
    }
    private consumerLogStatsLevel: LogLevel
    private onPartitionsRevoked?: (assignments: Assignment[]) => Promise<void>

    constructor(
        private config: KafkaConsumerConfig,
        rdKafkaConfig: RdKafkaConsumerConfig = {}
    ) {
        this.backgroundTask = []
        this.podName = process.env.HOSTNAME || hostname()
        this.lastBackgroundTaskCompletionTime = Date.now()
        // Generate unique consumer ID: pod + group + timestamp + random number (need timestamp/random number because multiple consumers per pod)
        this.consumerId = `${this.podName}-${this.config.groupId}-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`

        this.config.autoCommit ??= true
        this.config.autoOffsetStore ??= true
        this.config.callEachBatchWhenEmpty ??= false
        this.maxBackgroundTasks = defaultConfig.CONSUMER_MAX_BACKGROUND_TASKS
        this.fetchBatchSize = config.fetchBatchSize ?? defaultConfig.CONSUMER_BATCH_SIZE
        this.maxHealthHeartbeatIntervalMs =
            defaultConfig.CONSUMER_MAX_HEARTBEAT_INTERVAL_MS || MAX_HEALTH_HEARTBEAT_INTERVAL_MS
        this.consumerLoopStallThresholdMs = defaultConfig.CONSUMER_LOOP_STALL_THRESHOLD_MS
        this.consumerLogStatsLevel = defaultConfig.CONSUMER_LOG_STATS_LEVEL

        this.consumerConfig = stripClassicProtocolConfig({
            'client.id': hostname(),
            'security.protocol': 'plaintext',
            'metadata.broker.list': 'kafka:9092', // Overridden with KAFKA_CONSUMER_METADATA_BROKER_LIST
            log_level: 4, // WARN as the default
            'group.id': this.config.groupId,
            'session.timeout.ms': 30_000,
            'max.poll.interval.ms': 300_000,
            'max.partition.fetch.bytes': 1_048_576,
            'fetch.error.backoff.ms': 100,
            'fetch.message.max.bytes': 10_485_760,
            'fetch.wait.max.ms': 50,
            'queued.min.messages': 100000,
            'queued.max.messages.kbytes': 102400, // 1048576 is the default, we go smaller to reduce mem usage.
            'client.rack': defaultConfig.KAFKA_CLIENT_RACK, // Helps with cross-AZ traffic awareness and is not unique to the consumer
            'metadata.max.age.ms': 30000, // Refresh metadata every 30s - Relevant for leader loss (MSK Security Patches)
            'socket.timeout.ms': 30000,
            'enable.partition.eof': this.config.enablePartitionEof ?? true,
            // Only enable statistics when using loop-based health check
            ...(defaultConfig.CONSUMER_LOOP_BASED_HEALTH_CHECK
                ? { 'statistics.interval.ms': STATISTICS_INTERVAL_MS }
                : {}),
            // Custom settings and overrides - this is where most configuration overrides should be done
            // e.g. KAFKA_CONSUMER_ENABLE_PARTITION_EOF=false to override the default above
            ...getKafkaConfigFromEnv('CONSUMER'),
            // Finally any specifically given consumer config overrides
            ...rdKafkaConfig,
            // Below is config that we explicitly DO NOT want to be overrideable by env vars - i.e. things that would require code changes to change
            'partition.assignment.strategy': isTestEnv() ? 'roundrobin' : 'cooperative-sticky', // Roundrobin is used for testing to avoid flakiness caused by running librdkafka v2.2.0
            'enable.auto.offset.store': false, // NOTE: This is always false - we handle it using a custom function
            'enable.auto.commit': this.config.autoCommit,
            rebalance_cb: this.rebalanceCallback.bind(this),
            offset_commit_cb: true,
        })

        this.rdKafkaConsumer = this.createConsumer()
    }

    public getConfig(): ConsumerGlobalConfig {
        return {
            ...this.consumerConfig,
        }
    }

    public heartbeat(): void {
        // Can be called externally to update the heartbeat time and keep the consumer alive
        // This is maintained for backward compatibility with the legacy health check mechanism
        this.lastHeartbeatTime = Date.now()
    }

    /**
     * Say that this consumer waits on purpose, and does not stall.
     *
     * Separate from `heartbeat()` because it relaxes the loop stall detector, and most callers must
     * not do that. Two lanes drive `heartbeat()` from a `setInterval`, which keeps firing while a
     * batch is wedged on a promise that never settles, so moving the loop clock there would leave
     * such a pod reporting healthy forever.
     *
     * Call this only from a handler that sleeps by design, and only from its own await chain.
     */
    public reportDeliberateWait(): void {
        this.lastHeartbeatTime = Date.now()
        this.lastConsumerLoopTime = Date.now()
    }

    public isHealthy(): HealthCheckResult {
        // Use legacy heartbeat-based health check if feature flag is disabled
        if (!defaultConfig.CONSUMER_LOOP_BASED_HEALTH_CHECK) {
            // this is called as a readiness and a liveness probe
            const isWithinInterval = Date.now() - this.lastHeartbeatTime < this.maxHealthHeartbeatIntervalMs
            const isConnected = this.rdKafkaConsumer.isConnected()

            if (isConnected && isWithinInterval) {
                return new HealthCheckResultOk()
            } else {
                return new HealthCheckResultError('Consumer unhealthy', {
                    isConnected,
                    isWithinInterval,
                    lastHeartbeatTime: this.lastHeartbeatTime,
                    maxHealthHeartbeatIntervalMs: this.maxHealthHeartbeatIntervalMs,
                })
            }
        }

        // New loop-based health check implementation
        const details: Record<string, any> = {
            topic: this.config.topic,
            groupId: this.config.groupId,
            healthCheckMode: 'loop-based',
        }

        // 1. Basic connectivity check
        if (!this.rdKafkaConsumer.isConnected()) {
            return new HealthCheckResultError('Consumer not connected to Kafka broker', details)
        }

        // 2. Consumer loop liveness check (ensure loop is not stalled)
        const timeSinceLastLoop = Date.now() - this.lastConsumerLoopTime
        if (this.lastConsumerLoopTime > 0 && timeSinceLastLoop > this.consumerLoopStallThresholdMs) {
            return new HealthCheckResultError(
                `Consumer loop appears stalled (no activity for ${Math.round(timeSinceLastLoop / 1000)}s)`,
                {
                    ...details,
                    lastConsumerLoopTime: this.lastConsumerLoopTime,
                    timeSinceLastLoop,
                    threshold: this.consumerLoopStallThresholdMs,
                }
            )
        }

        // Build status message with warnings
        const warnings: string[] = []

        // 3. Check librdkafka internal state if available
        if (this.consumerState && this.consumerState !== 'up') {
            warnings.push(`Consumer state: ${this.consumerState}`)
            details.consumerState = this.consumerState
        }

        // 4. Check if statistics are being emitted (indicates librdkafka is responsive)
        if (this.lastStatsEmitTime > 0) {
            const timeSinceLastStats = Date.now() - this.lastStatsEmitTime
            // Allow for 3x the statistics interval as buffer
            if (timeSinceLastStats > STATISTICS_INTERVAL_MS * 3) {
                warnings.push(`Statistics not emitted for ${Math.round(timeSinceLastStats / 1000)}s`)
                details.lastStatsEmitTime = this.lastStatsEmitTime
                details.timeSinceLastStats = timeSinceLastStats
            }
        }

        // 5. Rebalancing is normal operation, note it but don't fail
        if (this.rebalanceCoordination.isRebalancing) {
            const duration = Date.now() - this.rebalanceCoordination.rebalanceStartTime
            warnings.push(`Rebalancing in progress (${Math.round(duration / 1000)}s)`)
            details.rebalancing = true
            details.rebalanceDuration = duration
        }

        // Add assignments info (but handle errors gracefully)
        try {
            const assignments = this.assignments()
            if (assignments.length > 0) {
                details.assignments = assignments.map((a) => ({ topic: a.topic, partition: a.partition }))
            }
        } catch (error) {
            // Consumer might be in an erroneous state during rebalancing
            details.assignmentError = error.message
        }

        // Return degraded if there are warnings, otherwise healthy
        if (warnings.length > 0) {
            return new HealthCheckResultDegraded(`Healthy with warnings: ${warnings.join(', ')}`, details)
        }

        return new HealthCheckResultOk()
    }

    public isShuttingDown(): boolean {
        return this.isStopping
    }

    public isRebalancing(): boolean {
        return this.rebalanceCoordination.isRebalancing
    }

    public assignments(): Assignment[] {
        return this.rdKafkaConsumer.isConnected() ? this.rdKafkaConsumer.assignments() : []
    }

    public offsetsStore(topicPartitionOffsets: TopicPartitionOffset[]): void {
        return this.rdKafkaConsumer.offsetsStore(topicPartitionOffsets)
    }

    public on: RdKafkaConsumer['on'] = (...args) => {
        // Delegate to the internal consumer
        return this.rdKafkaConsumer.on(...args)
    }

    public async queryWatermarkOffsets(topic: string, partition: number, timeout = 10000): Promise<[number, number]> {
        if (!this.rdKafkaConsumer.isConnected()) {
            throw new Error('Not connected')
        }

        const offsets = await promisifyCallback<WatermarkOffsets>((cb) =>
            this.rdKafkaConsumer.queryWatermarkOffsets(topic, partition, timeout, cb)
        ).catch((err) => {
            captureException(err)
            logger.error('🔥', 'Failed to query kafka watermark offsets', err)
            throw err
        })

        return [offsets.lowOffset, offsets.highOffset]
    }

    public async committedOffsets(timeout = 10000): Promise<TopicPartitionOffset[]> {
        if (!this.rdKafkaConsumer.isConnected()) {
            return []
        }

        return await promisifyCallback<TopicPartitionOffset[]>((cb) => this.rdKafkaConsumer.committed(timeout, cb))
    }

    public async getPartitionsForTopic(topic: string): Promise<PartitionMetadata[]> {
        if (!this.rdKafkaConsumer.isConnected()) {
            throw new Error('Not connected')
        }

        const meta = await promisifyCallback<Metadata>((cb) => this.rdKafkaConsumer.getMetadata({ topic }, cb)).catch(
            (err) => {
                captureException(err)
                logger.error('🔥', 'Failed to get partition metadata', err)
                throw err
            }
        )

        return meta.topics.find((x) => x.name === topic)?.partitions ?? []
    }

    public rebalanceCallback(err: LibrdKafkaError, assignments: Assignment[]): void {
        logger.info('🔁', 'kafka_consumer_rebalancing', { err, assignments })

        if (err.code === CODES.ERRORS.ERR__ASSIGN_PARTITIONS) {
            // Mark rebalancing as complete when partitions are assigned
            this.resetRebalanceCoordination()
            assignments.forEach((tp) => {
                kafkaConsumerAssignment.set(
                    {
                        topic_name: tp.topic,
                        partition_id: tp.partition.toString(),
                        pod: this.podName,
                        group_id: this.config.groupId,
                    },
                    1
                )
            })
            if (this.rdKafkaConsumer.rebalanceProtocol() === 'COOPERATIVE') {
                this.rdKafkaConsumer.incrementalAssign(assignments)
            } else {
                this.rdKafkaConsumer.assign(assignments)
            }
        } else if (err.code === CODES.ERRORS.ERR__REVOKE_PARTITIONS) {
            // Mark rebalancing as starting when partitions are revoked
            this.rebalanceCoordination.isRebalancing = true
            this.rebalanceCoordination.rebalanceStartTime = Date.now()
            logger.info('🔁', 'partition_revocation_starting', {
                backgroundTaskCount: this.backgroundTask.length,
                revokedPartitions: assignments.map((tp) => ({
                    topic: tp.topic,
                    partition: tp.partition,
                })),
            })

            // Handle background task coordination asynchronously
            if (this.backgroundTask.length > 0) {
                // Don't block the rebalance callback, but coordinate in the background
                Promise.all(this.backgroundTask.map((t) => t.promise))
                    .then(() => {
                        logger.info('🔁', 'background_tasks_completed_before_partition_revocation')
                        this.completeRevocation(assignments)
                    })
                    .catch((error) => {
                        logger.error('🔁', 'background_task_error_during_revocation', { error })
                        // Still proceed with revocation even if background tasks fail
                        this.completeRevocation(assignments)
                    })
            } else {
                // No background tasks, proceed immediately
                this.completeRevocation(assignments)
            }
        } else {
            // Ignore exceptions if we are not connected
            if (this.rdKafkaConsumer.isConnected()) {
                logger.error('🔥', 'kafka_consumer_rebalancing_error', { err })
                captureException(err)
            } else {
                logger.warn('🔥', 'kafka_consumer_rebalancing_error_while_not_connected', { err })
            }
        }
    }

    private completeRevocation(assignments: Assignment[]): void {
        try {
            if (this.rdKafkaConsumer.rebalanceProtocol() === 'COOPERATIVE') {
                this.rdKafkaConsumer.incrementalUnassign(assignments)
            } else {
                this.rdKafkaConsumer.unassign()
            }
            this.updateMetricsAfterRevocation(assignments)
        } finally {
            this.resetRebalanceCoordination()
        }
        Promise.resolve()
            .then(() => this.onPartitionsRevoked?.(assignments))
            .catch((error) => {
                logger.error('🔁', 'partition_revoked_handler_failed', { error: String(error) })
            })
    }

    private updateMetricsAfterRevocation(assignments: Assignment[]): void {
        assignments.forEach((tp) => {
            kafkaConsumerAssignment.set(
                {
                    topic_name: tp.topic,
                    partition_id: tp.partition.toString(),
                    pod: this.podName,
                    group_id: this.config.groupId,
                },
                0
            )
        })
    }

    private createConsumer(): RdKafkaConsumer {
        // auto.offset.reset is a topic-level property, and librdkafka ignores the global-config
        // form on our version (see node-rdkafka #984), so the resolved value, including any
        // caller override, must be mirrored into the explicit topic config. v2 does the same.
        const autoOffsetReset =
            (this.consumerConfig['auto.offset.reset' as keyof ConsumerGlobalConfig] as
                | 'earliest'
                | 'latest'
                | undefined) ?? 'earliest'
        const consumer = new RdKafkaConsumer(this.consumerConfig, {
            'auto.offset.reset': autoOffsetReset,
        })

        consumer.on('event.log', (log) => {
            logger.info('📝', 'librdkafka log', { log: log })
        })

        consumer.on('event.error', (error: LibrdKafkaError) => {
            logger.error('📝 librdkafka error', {
                message: error.message,
                code: error.code,
                errno: error.errno,
                origin: 'event.error',
                stack: error.stack,
            })
        })

        consumer.on('event.stats', (stats: any) => {
            // Parse the statistics JSON
            try {
                const parsedStats = parseJSON(stats.message)

                // Update internal health monitoring state
                this.lastStatsEmitTime = Date.now()
                // cgrp field only appears when consumer is part of a group
                this.consumerState = parsedStats.cgrp?.state || 'no-group'

                const brokerStats = parseBrokerStatistics(parsedStats)

                trackBrokerMetrics(brokerStats, this.config.groupId, this.consumerId)

                // Log key metrics for observability - only include cgrp fields if present
                const logData: any = {
                    rx_msgs: parsedStats.rxmsgs, // Total messages received
                    rx_bytes: parsedStats.rx_bytes || parsedStats.rxbytes, // Total bytes received
                    topics: Object.keys(parsedStats.topics || {}),
                    broker_count: brokerStats.size,
                    brokers: Array.from(brokerStats.entries()).map(([name, stats]) => ({
                        ...stats,
                        name,
                    })),
                }

                // Only add consumer group fields if cgrp exists
                if (parsedStats.cgrp) {
                    logData.consumer_group_state = parsedStats.cgrp.state
                    logData.rebalance_state = parsedStats.cgrp.join_state
                    logData.rebalance_age = parsedStats.cgrp.rebalance_age
                    logData.rebalance_cnt = parsedStats.cgrp.rebalance_cnt
                    logData.assignment_size = parsedStats.cgrp.assignment_size
                }

                logger[this.consumerLogStatsLevel]('📊', 'Kafka consumer statistics', logData)
            } catch (error) {
                logger.error('📊', 'Failed to parse consumer statistics', {
                    error: error instanceof Error ? error.message : String(error),
                    errorStack: error instanceof Error ? error.stack : undefined,
                })
            }
        })

        consumer.on('subscribed', (topics) => {
            logger.info('📝', 'librdkafka consumer subscribed', { topics, config: this.consumerConfig })
        })

        consumer.on('connection.failure', (error: LibrdKafkaError, metrics: ClientMetrics) => {
            logger.error('📝', 'librdkafka connection failure', { error, metrics, config: this.consumerConfig })
        })

        consumer.on('offset.commit', (error: LibrdKafkaError, topicPartitionOffsets: TopicPartitionOffset[]) => {
            if (error) {
                logger.warn('📝', 'librdkafka_offet_commit_error', { error, topicPartitionOffsets })
            } else {
                logger.debug('📝', 'librdkafka_offset_commit', { topicPartitionOffsets })
            }
        })

        return consumer
    }

    private storeOffsetsForMessages = (topicPartitionOffsetsToCommit: TopicPartitionOffset[]): void => {
        if (topicPartitionOffsetsToCommit.length > 0) {
            logger.debug('📝', 'Storing offsets', { topicPartitionOffsetsToCommit })
            try {
                this.rdKafkaConsumer.offsetsStore(topicPartitionOffsetsToCommit)
            } catch (e) {
                // NOTE: We don't throw here - this can happen if we were re-assigned partitions
                // and the offsets are no longer valid whilst processing a batch
                let assignedPartitions: Assignment[] | string = []
                try {
                    assignedPartitions = this.assignments()
                } catch (assignmentError) {
                    // Consumer might be in an erroneous state during rebalancing
                    assignedPartitions = `Error getting assignments: ${String(assignmentError)}`
                }
                logger.error('📝', 'Failed to store offsets', {
                    error: String(e),
                    assignedPartitions,
                    topicPartitionOffsetsToCommit,
                })
                captureException(e)
            }
        }
    }

    public async connect(
        eachBatch: (messages: Message[]) => Promise<{ backgroundTask?: Promise<any> } | void>,
        onPartitionsRevoked?: (assignments: Assignment[]) => Promise<void>
    ): Promise<void> {
        const { topic, groupId, callEachBatchWhenEmpty = false } = this.config
        this.onPartitionsRevoked = onPartitionsRevoked

        try {
            await promisifyCallback<Metadata>((cb) => this.rdKafkaConsumer.connect({}, cb))
            logger.info('📝', 'librdkafka consumer connected')
        } catch (error) {
            logger.error('⚠️', 'connect_error', { error: error })
            throw error
        }

        this.heartbeat() // Setup the heartbeat so we are healthy since connection is established

        // Initialize health monitoring state for new loop-based check
        this.lastConsumerLoopTime = Date.now()

        if (defaultConfig.CONSUMER_AUTO_CREATE_TOPICS) {
            // For hobby deploys we want to auto-create, but on cloud we don't
            await ensureTopicExists(this.consumerConfig, this.config.topic)
        }

        // The consumer has an internal pre-fetching queue that sequentially pools
        // each partition, with the consumerMaxWaitMs timeout. We want to read big
        // batches from this queue, but guarantee we are still running (with smaller
        // batches) if the queue is not full enough. batchingTimeoutMs is that
        // timeout, to return messages even if fetchBatchSize is not reached.
        this.rdKafkaConsumer.setDefaultConsumeTimeout(this.config.batchTimeoutMs || DEFAULT_BATCH_TIMEOUT_MS)
        this.rdKafkaConsumer.subscribe([this.config.topic])

        const startConsuming = async (): Promise<void> => {
            let lastConsumeTime = 0
            try {
                while (!this.isStopping) {
                    // Track that the consumer loop is alive
                    this.lastConsumerLoopTime = Date.now()
                    logger.debug('🔁', 'main_loop_consuming')

                    // If we're rebalancing, skip consuming to avoid processing messages
                    // during rebalancing when background tasks might be running
                    if (this.rebalanceCoordination.isRebalancing) {
                        if (
                            Date.now() - this.rebalanceCoordination.rebalanceStartTime >
                            this.rebalanceCoordination.rebalanceTimeoutMs
                        ) {
                            logger.error('🔁', 'rebalancing_timeout_forcing_recovery', {
                                rebalanceTimeoutMs: this.rebalanceCoordination.rebalanceTimeoutMs,
                                rebalanceStartTime: this.rebalanceCoordination.rebalanceStartTime,
                            })
                            this.rebalanceCoordination.isRebalancing = false
                        }
                        logger.info('🔁', 'main_loop_paused_for_rebalancing')
                        await new Promise((resolve) => setTimeout(resolve, 10)) // Small delay to avoid busy waiting
                        continue
                    }

                    const consumeStartTime = performance.now()
                    if (lastConsumeTime > 0) {
                        const intervalMs = consumeStartTime - lastConsumeTime
                        histogramKafkaConsumeInterval.labels({ topic, groupId }).observe(intervalMs)
                    }
                    lastConsumeTime = consumeStartTime
                    // TRICKY: We wrap this in a retry check. It seems that despite being connected and ready, the client can still have an undeterministic
                    // error when consuming, hence the retryIfRetriable.
                    const messages = await retryIfRetriable(() =>
                        promisifyCallback<Message[]>((cb) => this.rdKafkaConsumer.consume(this.fetchBatchSize, cb))
                    )

                    // After successfully pulling a batch, update heartbeat for backward compatibility
                    this.heartbeat()

                    gaugeBatchUtilization.labels({ groupId }).set(messages.length / this.fetchBatchSize)

                    logger.debug('🔁', 'main_loop_consumed', { messagesLength: messages.length })
                    histogramKafkaBatchSize.observe(messages.length)
                    histogramKafkaBatchSizeKb.observe(
                        messages.reduce((acc, m) => (m.value?.length ?? 0) + acc, 0) / 1024
                    )

                    if (!messages.length && !callEachBatchWhenEmpty) {
                        logger.debug('🔁', 'main_loop_empty_batch', { cause: 'empty' })
                        continue
                    }

                    const startProcessingTimeMs = new Date().valueOf()
                    const result = await eachBatch(messages)

                    const processingTimeMs = new Date().valueOf() - startProcessingTimeMs
                    consumedBatchDuration.labels({ topic, groupId }).observe(processingTimeMs)

                    const logSummary = `Processed ${messages.length} events in ${
                        Math.round(processingTimeMs / 10) / 100
                    }s`
                    if (processingTimeMs > SLOW_BATCH_PROCESSING_LOG_THRESHOLD_MS) {
                        logger.warn('🕒', `Slow batch: ${logSummary}, groupId: ${groupId}`)
                    }

                    // TRICKY: The commit logic needs to be aware of background work. If we were to just store offsets here,
                    // it would be hard to mix background work with non-background work.
                    // So we just create pretend work to simplify the rest of the logic
                    const rawBackgroundTask = result?.backgroundTask
                    const backgroundTask = rawBackgroundTask
                        ? instrumentFn(
                              {
                                  key: 'consumer_background_task',
                                  timeoutMs: defaultConfig.CONSUMER_BACKGROUND_TASK_TIMEOUT_MS,
                                  sendException: false,
                              },
                              () => rawBackgroundTask
                          )
                        : Promise.resolve()
                    const stopBackgroundTaskTimer = rawBackgroundTask
                        ? consumedBatchBackgroundDuration.startTimer({
                              topic: this.config.topic,
                              groupId: this.config.groupId,
                          })
                        : undefined
                    const taskCreatedAt = Date.now()
                    // Pull out the offsets to commit from the messages so we can release the messages reference
                    const topicPartitionOffsetsToCommit = findOffsetsToCommit(messages)

                    void backgroundTask.finally(async () => {
                        // Track that we made progress
                        this.lastBackgroundTaskCompletionTime = Date.now()

                        stopBackgroundTaskTimer?.()

                        // First of all clear ourselves from the queue
                        const index = this.backgroundTask.findIndex((t) => t.promise === backgroundTask)

                        // CRITICAL: If task not found, this indicates some bigger problem
                        if (index < 0) {
                            captureException(new Error('Background task not found in array during cleanup'))
                        }

                        // TRICKY: We need to wait for all promises ahead of us in the queue before we store the offsets
                        // Important: capture the promises BEFORE removing the task, as the array changes after splice
                        if (index >= 0) {
                            // Task found - capture promises to wait for, then remove the task
                            const promisesToWait = this.backgroundTask.slice(0, index).map((t) => t.promise)
                            this.backgroundTask.splice(index, 1)
                            await Promise.all(promisesToWait)
                        }

                        if (this.config.autoCommit && this.config.autoOffsetStore) {
                            this.storeOffsetsForMessages(topicPartitionOffsetsToCommit)
                        }
                    })

                    // At first we just add the background work to the queue with metadata
                    this.backgroundTask.push({
                        promise: backgroundTask,
                        createdAt: taskCreatedAt,
                    })

                    // If we have too much "backpressure" we need to await one of the background tasks. We await the oldest one on purpose
                    if (this.backgroundTask.length >= this.maxBackgroundTasks) {
                        const stopTimer = consumedBatchBackpressureDuration.startTimer({
                            topic: this.config.topic,
                            groupId: this.config.groupId,
                        })
                        // If we have more than the max, we need to await one
                        await instrumentFn(
                            { key: 'consumer_backpressure_wait', timeoutMs: 30_000, sendException: false },
                            () => this.backgroundTask[0].promise
                        )
                        stopTimer()
                    }
                }

                // Once we are stopping, make sure that we wait for all background work to finish
                await Promise.all(this.backgroundTask.map((t) => t.promise))
            } catch (error) {
                logger.error('🔁', 'main_loop_error', {
                    error: String(error),
                    errorCode: (error as any)?.code,
                    errorOrigin: (error as any)?.origin,
                    isFatal: (error as any)?.isFatal,
                    isRetriable: (error as any)?.isRetriable,
                })
                throw error
            } finally {
                logger.info('🔁', 'main_loop_stopping')

                // Finally, disconnect from the broker. If stored offsets have changed via
                // `storeOffsetsForMessages` above, they will be committed before shutdown (so long
                // as this consumer is still part of the group).
                await this.disconnectConsumer()
                logger.info('🔁', 'Disconnected node-rdkafka consumer')
            }
        }

        this.consumerLoop = startConsuming().catch((error) => {
            logger.error('🔁', 'consumer_loop_error', {
                error: String(error),
                errorCode: (error as any)?.code,
                errorOrigin: (error as any)?.origin,
                isFatal: (error as any)?.isFatal,
                isRetriable: (error as any)?.isRetriable,
                config: this.config,
                consumerConfig: this.consumerConfig,
            })
            // We re-throw the error as that way it will be caught in server.ts and trigger a full shutdown
            throw error
        })
    }

    public async disconnect(): Promise<void> {
        if (this.isStopping) {
            return
        }
        // Mark as stopping - this will also essentially stop the consumer loop
        this.isStopping = true

        // Wait for background tasks to complete before disconnecting
        logger.info('🔁', 'waiting_for_background_tasks_before_disconnect', {
            backgroundTaskCount: this.backgroundTask.length,
        })
        await Promise.all(this.backgroundTask.map((t) => t.promise))

        logger.info('🔁', 'background_tasks_completed_proceeding_with_disconnect')

        // Allow the in progress consumer loop to finish if possible
        if (this.consumerLoop) {
            await this.consumerLoop.catch((error) => {
                logger.error('🔁', 'failed to stop consumer loop safely. Continuing shutdown', {
                    error: String(error),
                    errorCode: (error as any)?.code,
                    errorOrigin: (error as any)?.origin,
                    isFatal: (error as any)?.isFatal,
                    isRetriable: (error as any)?.isRetriable,
                    config: this.config,
                    consumerConfig: this.consumerConfig,
                })
            })
        }

        await this.disconnectConsumer()
    }

    private async disconnectConsumer(): Promise<void> {
        if (this.rdKafkaConsumer.isConnected()) {
            logger.info('📝', 'Disconnecting consumer...')
            await new Promise<void>((res, rej) => this.rdKafkaConsumer.disconnect((e) => (e ? rej(e) : res())))
            logger.info('📝', 'Disconnected consumer!')
        }
    }

    private resetRebalanceCoordination(): void {
        this.rebalanceCoordination.isRebalancing = false
        this.rebalanceCoordination.rebalanceStartTime = 0
    }
}

export const parseKafkaHeaders = (headers?: MessageHeader[]): Record<string, string> => {
    // Kafka headers come from librdkafka as an array of objects with keys value pairs per header.
    // It's a confusing format so we simplify it to a record.

    const result: Record<string, string> = {}

    headers?.forEach((header) => {
        Object.keys(header).forEach((key) => {
            result[key] = header[key].toString()
        })
    })

    return result
}

export const parseEventHeaders = (headers?: MessageHeader[]): EventHeaders => {
    // Kafka headers come from librdkafka as an array of objects with keys value pairs per header.
    // We extract the specific headers we care about into a structured format.

    const result: EventHeaders = {
        force_disable_person_processing: false,
        historical_migration: false,
        skip_heatmap_processing: false,
    }

    headers?.forEach((header) => {
        Object.keys(header).forEach((key) => {
            const value = header[key].toString()
            if (key === 'token') {
                result.token = sanitizeString(value)
            } else if (key === 'distinct_id') {
                result.distinct_id = sanitizeString(value)
            } else if (key === 'session_id') {
                result.session_id = normalizeSessionId(sanitizeString(value))
            } else if (key === 'timestamp') {
                result.timestamp = value
            } else if (key === 'event') {
                result.event = value
            } else if (key === 'uuid') {
                result.uuid = value
            } else if (key === 'force_disable_person_processing') {
                result.force_disable_person_processing = value === 'true'
            } else if (key === 'now') {
                const parsed = new Date(value)
                if (!isNaN(parsed.getTime())) {
                    result.now = parsed
                }
            } else if (key === 'historical_migration') {
                result.historical_migration = value === 'true'
            } else if (key === 'skip_heatmap_processing') {
                result.skip_heatmap_processing = value === 'true'
            } else if (key === 'redirect-original-key') {
                result.redirect_original_key = value
            }
        })
    })

    // Track comprehensive header status metrics
    const trackedHeaders = [
        'token',
        'distinct_id',
        'session_id',
        'timestamp',
        'event',
        'uuid',
        'now',
        'force_disable_person_processing',
        'historical_migration',
        'skip_heatmap_processing',
    ] as const
    trackedHeaders.forEach((header) => {
        const status = result[header] ? 'present' : 'absent'
        kafkaHeaderStatusCounter.labels(header, status).inc()
    })

    return result
}