Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.5 KB
Raw Blame
import {
    KAFKA_APP_METRICS_2,
    KAFKA_CLICKHOUSE_TOPHOG,
    KAFKA_ERROR_TRACKING_INGESTION,
    KAFKA_ERROR_TRACKING_INGESTION_DLQ,
    KAFKA_ERROR_TRACKING_INGESTION_OVERFLOW,
    KAFKA_EVENTS_JSON,
    KAFKA_INGESTION_WARNINGS,
    KAFKA_LOG_ENTRIES,
} from '~/common/config/kafka-topics'
import {
    INGESTION_DOWNSTREAM_PRODUCER,
    INGESTION_UPSTREAM_PRODUCER,
    type ProducerName,
} from '~/ingestion/common/outputs/producers'
import { IngestionLane, IngestionOverflowMode } from '~/ingestion/config'

export type ErrorTrackingConsumerConfig = {
    ERROR_TRACKING_CONSUMER_GROUP_ID: string
    ERROR_TRACKING_CONSUMER_CONSUME_TOPIC: string
    ERROR_TRACKING_CONSUMER_DLQ_TOPIC: string
    ERROR_TRACKING_CONSUMER_OVERFLOW_TOPIC: string
    ERROR_TRACKING_CONSUMER_OUTPUT_TOPIC: string
    ERROR_TRACKING_CYMBAL_BASE_URL: string
    ERROR_TRACKING_CYMBAL_TIMEOUT_MS: number

    /** Token bucket capacity for rate limiting (events per token:distinct_id) */
    ERROR_TRACKING_OVERFLOW_BUCKET_CAPACITY: number
    /** Token bucket replenish rate (events per second) */
    ERROR_TRACKING_OVERFLOW_BUCKET_REPLENISH_RATE: number
    /** TTL in seconds for Redis overflow flags */
    ERROR_TRACKING_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS: number
    /** TTL in seconds for local cache entries */
    ERROR_TRACKING_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS: number
    /**
     * When true (default, matching the previous hardcoded behavior), redirects
     * to the overflow lane keep the event's original partition key. When
     * false, the overflow producer emits with a null key and Kafka spreads
     * events across overflow-topic partitions. Cymbal cache locality is
     * enforced one layer down — by team_id consistent hashing inside
     * `CymbalClient` — so the partition key on the overflow lane doesn't
     * affect symbolication cache hits.
     */
    ERROR_TRACKING_OVERFLOW_PRESERVE_PARTITION_LOCALITY: boolean

    /** Max HTTP body size in bytes per Cymbal API request. Used to proactively
     *  split large batches before they hit Cymbal's body limit. */
    ERROR_TRACKING_CYMBAL_MAX_BODY_BYTES: number

    /** Pipeline name for metrics labeling */
    INGESTION_PIPELINE: string | null
    /** Lane identifier (main, overflow) for metrics labeling */
    INGESTION_LANE: IngestionLane | null
    /** How this consumer participates in overflow handling (redirect / consume / disabled) */
    INGESTION_OVERFLOW_MODE: IngestionOverflowMode
}

export function getDefaultErrorTrackingConsumerConfig(): ErrorTrackingConsumerConfig {
    return {
        ERROR_TRACKING_CONSUMER_GROUP_ID: 'ingestion-errortracking',
        ERROR_TRACKING_CONSUMER_CONSUME_TOPIC: KAFKA_ERROR_TRACKING_INGESTION,
        ERROR_TRACKING_CONSUMER_DLQ_TOPIC: KAFKA_ERROR_TRACKING_INGESTION_DLQ,
        ERROR_TRACKING_CONSUMER_OVERFLOW_TOPIC: KAFKA_ERROR_TRACKING_INGESTION_OVERFLOW,
        ERROR_TRACKING_CONSUMER_OUTPUT_TOPIC: KAFKA_EVENTS_JSON,
        ERROR_TRACKING_CYMBAL_BASE_URL: 'http://localhost:3302',
        ERROR_TRACKING_CYMBAL_TIMEOUT_MS: 15000,
        ERROR_TRACKING_OVERFLOW_BUCKET_CAPACITY: 1000,
        ERROR_TRACKING_OVERFLOW_BUCKET_REPLENISH_RATE: 1.0,
        ERROR_TRACKING_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS: 300, // 5 minutes
        ERROR_TRACKING_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS: 60, // 1 minute
        ERROR_TRACKING_OVERFLOW_PRESERVE_PARTITION_LOCALITY: true,
        ERROR_TRACKING_CYMBAL_MAX_BODY_BYTES: 1_800_000,
        INGESTION_PIPELINE: null,
        INGESTION_LANE: null,
        INGESTION_OVERFLOW_MODE: 'disabled',
    }
}

/** Config type for all error tracking output keys. */
export type ErrorTrackingOutputsConfig = {
    ERROR_TRACKING_CONSUMER_OUTPUT_TOPIC: string
    ERROR_TRACKING_OUTPUT_EVENTS_PRODUCER: ProducerName

    ERROR_TRACKING_OUTPUT_INGESTION_WARNINGS_TOPIC: string
    ERROR_TRACKING_OUTPUT_INGESTION_WARNINGS_PRODUCER: ProducerName

    ERROR_TRACKING_CONSUMER_DLQ_TOPIC: string
    ERROR_TRACKING_OUTPUT_DLQ_PRODUCER: ProducerName

    ERROR_TRACKING_CONSUMER_OVERFLOW_TOPIC: string
    ERROR_TRACKING_OUTPUT_OVERFLOW_PRODUCER: ProducerName

    ERROR_TRACKING_OUTPUT_APP_METRICS_TOPIC: string
    ERROR_TRACKING_OUTPUT_APP_METRICS_PRODUCER: ProducerName

    ERROR_TRACKING_OUTPUT_LOG_ENTRIES_TOPIC: string
    ERROR_TRACKING_OUTPUT_LOG_ENTRIES_PRODUCER: ProducerName

    ERROR_TRACKING_OUTPUT_TOPHOG_TOPIC: string
    ERROR_TRACKING_OUTPUT_TOPHOG_PRODUCER: ProducerName
}

export function getDefaultErrorTrackingOutputsConfig(): ErrorTrackingOutputsConfig {
    return {
        ERROR_TRACKING_CONSUMER_OUTPUT_TOPIC: KAFKA_EVENTS_JSON,
        ERROR_TRACKING_OUTPUT_EVENTS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
        ERROR_TRACKING_OUTPUT_INGESTION_WARNINGS_TOPIC: KAFKA_INGESTION_WARNINGS,
        ERROR_TRACKING_OUTPUT_INGESTION_WARNINGS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
        ERROR_TRACKING_CONSUMER_DLQ_TOPIC: KAFKA_ERROR_TRACKING_INGESTION_DLQ,
        ERROR_TRACKING_OUTPUT_DLQ_PRODUCER: INGESTION_UPSTREAM_PRODUCER,
        ERROR_TRACKING_CONSUMER_OVERFLOW_TOPIC: KAFKA_ERROR_TRACKING_INGESTION_OVERFLOW,
        ERROR_TRACKING_OUTPUT_OVERFLOW_PRODUCER: INGESTION_UPSTREAM_PRODUCER,
        ERROR_TRACKING_OUTPUT_APP_METRICS_TOPIC: KAFKA_APP_METRICS_2,
        ERROR_TRACKING_OUTPUT_APP_METRICS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
        ERROR_TRACKING_OUTPUT_LOG_ENTRIES_TOPIC: KAFKA_LOG_ENTRIES,
        ERROR_TRACKING_OUTPUT_LOG_ENTRIES_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
        ERROR_TRACKING_OUTPUT_TOPHOG_TOPIC: KAFKA_CLICKHOUSE_TOPHOG,
        ERROR_TRACKING_OUTPUT_TOPHOG_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
    }
}