Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.9 KB
Raw Blame
import { Counter, Histogram } from 'prom-client'

import { logger } from '../utils/logger'

// Thresholds for broker state monitoring (in microseconds)
const SSL_HANDSHAKE_TIMEOUT_THRESHOLD_US = 5_000_000
const AUTH_TIMEOUT_THRESHOLD_US = 10_000_000

export const kafkaBrokerRtt = new Histogram({
    name: 'kafka_broker_rtt_ms',
    help: 'Round trip time to broker in milliseconds',
    labelNames: ['broker_id', 'broker_name', 'consumer_group'],
    buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],
})

export const kafkaBrokerSslHandshakeFailures = new Counter({
    name: 'kafka_broker_ssl_handshake_failures_total',
    help: 'Total number of SSL handshake failures',
    labelNames: ['broker_id', 'broker_name', 'consumer_group', 'error'],
})

export const kafkaBrokerAuthFailures = new Counter({
    name: 'kafka_broker_auth_failures_total',
    help: 'Total number of authentication failures',
    labelNames: ['broker_id', 'broker_name', 'consumer_group'],
})

// interfaces pulled from here:
// https://docs.confluent.io/platform/current/clients/librdkafka/html/md_STATISTICS.html

export interface BrokerStats {
    name: string
    nodeid: number
    nodename: string
    source: string
    state: string
    // time since last broker state change (in microseconds)
    stateage: number
    outbuf_cnt: number
    outbuf_msg_cnt: number
    waitresp_cnt: number
    waitresp_msg_cnt: number
    tx: number
    txbytes: number
    txerrs: number
    txretries: number
    txidle: number
    req_timeouts: number
    rx: number
    rxbytes: number
    rxerrs: number
    rxcorriderrs: number
    rxpartial: number
    rxidle: number
    zbuf_grow: number
    buf_grow: number
    wakeups: number
    // number of connections attempts, including successful and failed, and name resolution failures
    connects: number
    // number of disconnects (triggered by broker, network, load-balancer, etc)
    disconnects: number
    int_latency: {
        min: number
        max: number
        avg: number
        sum: number
        stddev: number
        p50: number
        p75: number
        p90: number
        p95: number
        p99: number
        p99_99: number
        outofrange: number
        hdrsize: number
        cnt: number
    }
    outbuf_latency: {
        min: number
        max: number
        avg: number
        sum: number
        stddev: number
        p50: number
        p75: number
        p90: number
        p95: number
        p99: number
        p99_99: number
        outofrange: number
        hdrsize: number
        cnt: number
    }
    rtt: {
        min: number
        max: number
        avg: number
        sum: number
        stddev: number
        p50: number
        p75: number
        p90: number
        p95: number
        p99: number
        p99_99: number
        outofrange: number
        hdrsize: number
        cnt: number
    }
    throttle: {
        min: number
        max: number
        avg: number
        sum: number
        stddev: number
        p50: number
        p75: number
        p90: number
        p95: number
        p99: number
        p99_99: number
        outofrange: number
        hdrsize: number
        cnt: number
    }
    toppars?: {
        [topicPartition: string]: {
            topic: string
            partition: number
        }
    }
}

export function parseBrokerStatistics(stats: any): Map<string, BrokerStats> {
    const brokerStats = new Map<string, BrokerStats>()

    if (!stats.brokers) {
        return brokerStats
    }

    for (const [brokerName, broker] of Object.entries(stats.brokers)) {
        if (typeof broker === 'object' && broker !== null) {
            brokerStats.set(brokerName, broker as BrokerStats)
        }
    }
    return brokerStats
}

export function trackBrokerMetrics(
    brokerStats: Map<string, BrokerStats>,
    consumerGroup: string,
    consumerId: string
): void {
    for (const [brokerName, stats] of brokerStats) {
        const labels = {
            broker_id: String(stats.nodeid),
            broker_name: brokerName,
            consumer_group: consumerGroup,
        }

        if (stats.rtt?.avg) {
            kafkaBrokerRtt.observe(labels, stats.rtt.avg)
        }

        // Check for broker connection issues
        if (stats.state === 'DOWN' || stats.state === 'CONNECT') {
            logger.error('Broker connection issue', {
                ...labels,
                consumer_id: consumerId,
                state: stats.state,
                stateage: stats.stateage,
                connects: stats.connects,
                disconnects: stats.disconnects,
                txerrs: stats.txerrs,
                rxerrs: stats.rxerrs,
            })
        }

        // ssl handshake issues
        if (stats.state === 'SSL_HANDSHAKE' && stats.stateage > SSL_HANDSHAKE_TIMEOUT_THRESHOLD_US) {
            kafkaBrokerSslHandshakeFailures.inc({
                ...labels,
                error: 'ssl_handshake_timeout',
            })
            logger.error('SSL handshake stuck', {
                ...labels,
                consumer_id: consumerId,
                stateage_ms: stats.stateage,
            })
        }

        // auth issues
        if ((stats.state === 'AUTH' || stats.state === 'SASL_AUTH') && stats.stateage > AUTH_TIMEOUT_THRESHOLD_US) {
            kafkaBrokerAuthFailures.inc({
                ...labels,
            })
            logger.error('Authentication stuck', {
                ...labels,
                consumer_id: consumerId,
                stateage_ms: stats.stateage,
            })
        }

        // request timeouts
        if (stats.req_timeouts > 0) {
            logger.warn('Broker request timeouts', {
                ...labels,
                consumer_id: consumerId,
                req_timeouts: stats.req_timeouts,
                waitresp_cnt: stats.waitresp_cnt,
                waitresp_msg_cnt: stats.waitresp_msg_cnt,
            })
        }
    }
}