Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.0 KB
Raw Blame
use std::time::Duration;

use common_kafka::config::KafkaConfig;
use envconfig::Envconfig;

use crate::counters::CounterConfig;

/// WarpStream's recommended librdkafka producer settings.
/// <https://docs.warpstream.com/warpstream/kafka/configure-kafka-client/tuning-for-performance>
const BATCH_SIZE: u32 = 16_000_000;
const BATCH_NUM_MESSAGES: u32 = 100_000;
/// librdkafka caps a produce request at this, so it has to exceed `BATCH_SIZE`.
const MESSAGE_MAX_BYTES: u32 = 64_000_000;
const MAX_IN_FLIGHT_PER_CONNECTION: u32 = 1_000_000;
const METADATA_REFRESH_INTERVAL_MS: u32 = 60_000;

const _: () = assert!(MESSAGE_MAX_BYTES > BATCH_SIZE);

#[derive(Envconfig, Clone)]
pub struct Config {
    #[envconfig(from = "USAGE_INGESTION_GRPC_ADDRESS", default = "0.0.0.0:7143")]
    pub grpc_address: String,
    #[envconfig(from = "USAGE_INGESTION_METRICS_ADDRESS", default = "0.0.0.0:7144")]
    pub metrics_address: String,
    #[envconfig(from = "USAGE_INGESTION_DATABASE_URL")]
    pub database_url: String,
    #[envconfig(from = "KAFKA_HOSTS", default = "localhost:9092")]
    pub kafka_hosts: String,
    #[envconfig(from = "KAFKA_TLS", default = "false")]
    pub kafka_tls: bool,
    /// Only "none", "gzip", "snappy" and "lz4" work. "zstd" needs an rdkafka feature the
    /// workspace does not enable, so librdkafka refuses it when it builds the producer.
    #[envconfig(from = "KAFKA_COMPRESSION_CODEC", default = "lz4")]
    pub kafka_compression_codec: String,
    /// These records never fill a batch, so linger decides when a produce goes out and lands
    /// in ingest latency. Lower it if that matters more than batching.
    #[envconfig(from = "KAFKA_PRODUCER_LINGER_MS", default = "100")]
    pub kafka_producer_linger_ms: u32,
    #[envconfig(from = "USAGE_INGESTION_MAX_BATCH_SIZE", default = "500")]
    pub max_batch_size: usize,
    /// Empty keeps the Redis projection disabled, which is the safe default for existing deployments.
    #[envconfig(from = "USAGE_INGESTION_REDIS_URL", default = "")]
    pub redis_url: String,
    #[envconfig(from = "USAGE_INGESTION_REDIS_FLUSH_INTERVAL_SECONDS", default = "15")]
    pub redis_flush_interval_seconds: u64,
    #[envconfig(from = "USAGE_INGESTION_REDIS_FLUSH_CONCURRENCY", default = "16")]
    pub redis_flush_concurrency: usize,
    // Overridable so a test environment can use the suffixed topic its Kafka engine table reads.
    #[envconfig(
        from = "USAGE_INGESTION_TOPIC",
        default = "clickhouse_billing_usage_records"
    )]
    pub topic: String,
    /// Maximum age of a gRPC connection in seconds before the server sends GOAWAY.
    /// Producers reconnect transparently, which restaggers them across the pods.
    /// 0 = disabled (connections live indefinitely).
    /// Shorter than personhog's 300 because a producer holds one connection and this fleet
    /// runs near its CPU request when the load lands unevenly.
    #[envconfig(from = "USAGE_INGESTION_GRPC_MAX_CONNECTION_AGE_SECS", default = "60")]
    pub grpc_max_connection_age_secs: u64,
}

impl Config {
    pub fn validate(&self) -> Result<(), String> {
        if self.max_batch_size == 0 || self.max_batch_size > 5_000 {
            return Err("USAGE_INGESTION_MAX_BATCH_SIZE must be between 1 and 5000".to_string());
        }
        if self.redis_flush_interval_seconds == 0 {
            return Err(
                "USAGE_INGESTION_REDIS_FLUSH_INTERVAL_SECONDS must be positive".to_string(),
            );
        }
        if self.redis_flush_concurrency == 0 {
            return Err("USAGE_INGESTION_REDIS_FLUSH_CONCURRENCY must be positive".to_string());
        }
        // A few seconds would make every producer spend its time reconnecting.
        if self.grpc_max_connection_age_secs > 0 && self.grpc_max_connection_age_secs < 10 {
            return Err(
                "USAGE_INGESTION_GRPC_MAX_CONNECTION_AGE_SECS must be 0 or at least 10".to_string(),
            );
        }
        Ok(())
    }

    pub fn redis_counter_config(&self) -> CounterConfig {
        CounterConfig {
            flush_concurrency: self.redis_flush_concurrency,
        }
    }

    pub fn grpc_max_connection_age(&self) -> Option<Duration> {
        if self.grpc_max_connection_age_secs == 0 {
            None
        } else {
            Some(Duration::from_secs(self.grpc_max_connection_age_secs))
        }
    }

    /// A setting dropped from here reverts to a `KafkaConfig` default instead of failing.
    pub fn kafka_config(&self) -> KafkaConfig {
        KafkaConfig {
            kafka_hosts: self.kafka_hosts.clone(),
            kafka_tls: self.kafka_tls,
            kafka_client_id: "usage-ingestion".to_string(),
            kafka_compression_codec: self.kafka_compression_codec.clone(),
            kafka_producer_linger_ms: self.kafka_producer_linger_ms,
            kafka_producer_batch_size: Some(BATCH_SIZE),
            kafka_producer_batch_num_messages: Some(BATCH_NUM_MESSAGES),
            kafka_producer_message_max_bytes: Some(MESSAGE_MAX_BYTES),
            kafka_producer_max_in_flight_requests_per_connection: Some(
                MAX_IN_FLIGHT_PER_CONNECTION,
            ),
            kafka_producer_topic_metadata_refresh_interval_ms: Some(METADATA_REFRESH_INTERVAL_MS),
            // WarpStream serves a different agent per connection, which collapses throughput
            // when librdkafka pins an idempotent producer's sequence numbers to one.
            kafka_producer_enable_idempotence: Some(false),
            // acks and the partitioner stay at librdkafka's defaults, which match WarpStream's
            // advice: acks=all keeps a billing record durable, and these records carry no key.
            ..Default::default()
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn config() -> Config {
        Config {
            grpc_address: "0.0.0.0:7143".to_string(),
            metrics_address: "0.0.0.0:7144".to_string(),
            database_url: "postgres://localhost/test".to_string(),
            kafka_hosts: "localhost:9092".to_string(),
            kafka_tls: false,
            kafka_compression_codec: "lz4".to_string(),
            kafka_producer_linger_ms: 100,
            max_batch_size: 500,
            redis_url: String::new(),
            redis_flush_interval_seconds: 15,
            redis_flush_concurrency: 16,
            topic: "clickhouse_billing_usage_records".to_string(),
            grpc_max_connection_age_secs: 60,
        }
    }

    #[test]
    fn the_producer_follows_the_warpstream_recommendations() {
        let kafka = config().kafka_config();

        assert_eq!(kafka.kafka_compression_codec, "lz4");
        assert_eq!(kafka.kafka_producer_linger_ms, 100);
        assert_eq!(kafka.kafka_producer_batch_size, Some(BATCH_SIZE));
        assert_eq!(
            kafka.kafka_producer_batch_num_messages,
            Some(BATCH_NUM_MESSAGES)
        );
        assert_eq!(
            kafka.kafka_producer_message_max_bytes,
            Some(MESSAGE_MAX_BYTES)
        );
        assert_eq!(
            kafka.kafka_producer_max_in_flight_requests_per_connection,
            Some(MAX_IN_FLIGHT_PER_CONNECTION)
        );
        assert_eq!(
            kafka.kafka_producer_topic_metadata_refresh_interval_ms,
            Some(METADATA_REFRESH_INTERVAL_MS)
        );
        assert_eq!(kafka.kafka_producer_enable_idempotence, Some(false));
    }

    #[test]
    fn the_environment_overrides_compression_and_linger() {
        let config = Config {
            kafka_compression_codec: "none".to_string(),
            kafka_producer_linger_ms: 5,
            ..config()
        };

        let kafka = config.kafka_config();

        assert_eq!(kafka.kafka_compression_codec, "none");
        assert_eq!(kafka.kafka_producer_linger_ms, 5);
    }

    #[test]
    fn redis_counter_configuration_requires_positive_values() {
        let config = Config {
            redis_flush_concurrency: 0,
            ..config()
        };

        assert_eq!(
            config.validate(),
            Err("USAGE_INGESTION_REDIS_FLUSH_CONCURRENCY must be positive".to_string())
        );
    }
}