:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::time::Duration;
use common_kafka::config::KafkaConfig;
use envconfig::Envconfig;
use crate::counters::CounterConfig;
/// WarpStream's recommended librdkafka producer settings.
/// <https://docs.warpstream.com/warpstream/kafka/configure-kafka-client/tuning-for-performance>
const BATCH_SIZE: u32 = 16_000_000;
const BATCH_NUM_MESSAGES: u32 = 100_000;
/// librdkafka caps a produce request at this, so it has to exceed `BATCH_SIZE`.
const MESSAGE_MAX_BYTES: u32 = 64_000_000;
const MAX_IN_FLIGHT_PER_CONNECTION: u32 = 1_000_000;
const METADATA_REFRESH_INTERVAL_MS: u32 = 60_000;
const _: () = assert!(MESSAGE_MAX_BYTES > BATCH_SIZE);
#[derive(Envconfig, Clone)]
pub struct Config {
#[envconfig(from = "USAGE_INGESTION_GRPC_ADDRESS", default = "0.0.0.0:7143")]
pub grpc_address: String,
#[envconfig(from = "USAGE_INGESTION_METRICS_ADDRESS", default = "0.0.0.0:7144")]
pub metrics_address: String,
#[envconfig(from = "USAGE_INGESTION_DATABASE_URL")]
pub database_url: String,
#[envconfig(from = "KAFKA_HOSTS", default = "localhost:9092")]
pub kafka_hosts: String,
#[envconfig(from = "KAFKA_TLS", default = "false")]
pub kafka_tls: bool,
/// Only "none", "gzip", "snappy" and "lz4" work. "zstd" needs an rdkafka feature the
/// workspace does not enable, so librdkafka refuses it when it builds the producer.
#[envconfig(from = "KAFKA_COMPRESSION_CODEC", default = "lz4")]
pub kafka_compression_codec: String,
/// These records never fill a batch, so linger decides when a produce goes out and lands
/// in ingest latency. Lower it if that matters more than batching.
#[envconfig(from = "KAFKA_PRODUCER_LINGER_MS", default = "100")]
pub kafka_producer_linger_ms: u32,
#[envconfig(from = "USAGE_INGESTION_MAX_BATCH_SIZE", default = "500")]
pub max_batch_size: usize,
/// Empty keeps the Redis projection disabled, which is the safe default for existing deployments.
#[envconfig(from = "USAGE_INGESTION_REDIS_URL", default = "")]
pub redis_url: String,
#[envconfig(from = "USAGE_INGESTION_REDIS_FLUSH_INTERVAL_SECONDS", default = "15")]
pub redis_flush_interval_seconds: u64,
#[envconfig(from = "USAGE_INGESTION_REDIS_FLUSH_CONCURRENCY", default = "16")]
pub redis_flush_concurrency: usize,
// Overridable so a test environment can use the suffixed topic its Kafka engine table reads.
#[envconfig(
from = "USAGE_INGESTION_TOPIC",
default = "clickhouse_billing_usage_records"
)]
pub topic: String,
/// Maximum age of a gRPC connection in seconds before the server sends GOAWAY.
/// Producers reconnect transparently, which restaggers them across the pods.
/// 0 = disabled (connections live indefinitely).
/// Shorter than personhog's 300 because a producer holds one connection and this fleet
/// runs near its CPU request when the load lands unevenly.
#[envconfig(from = "USAGE_INGESTION_GRPC_MAX_CONNECTION_AGE_SECS", default = "60")]
pub grpc_max_connection_age_secs: u64,
}
impl Config {
pub fn validate(&self) -> Result<(), String> {
if self.max_batch_size == 0 || self.max_batch_size > 5_000 {
return Err("USAGE_INGESTION_MAX_BATCH_SIZE must be between 1 and 5000".to_string());
}
if self.redis_flush_interval_seconds == 0 {
return Err(
"USAGE_INGESTION_REDIS_FLUSH_INTERVAL_SECONDS must be positive".to_string(),
);
}
if self.redis_flush_concurrency == 0 {
return Err("USAGE_INGESTION_REDIS_FLUSH_CONCURRENCY must be positive".to_string());
}
// A few seconds would make every producer spend its time reconnecting.
if self.grpc_max_connection_age_secs > 0 && self.grpc_max_connection_age_secs < 10 {
return Err(
"USAGE_INGESTION_GRPC_MAX_CONNECTION_AGE_SECS must be 0 or at least 10".to_string(),
);
}
Ok(())
}
pub fn redis_counter_config(&self) -> CounterConfig {
CounterConfig {
flush_concurrency: self.redis_flush_concurrency,
}
}
pub fn grpc_max_connection_age(&self) -> Option<Duration> {
if self.grpc_max_connection_age_secs == 0 {
None
} else {
Some(Duration::from_secs(self.grpc_max_connection_age_secs))
}
}
/// A setting dropped from here reverts to a `KafkaConfig` default instead of failing.
pub fn kafka_config(&self) -> KafkaConfig {
KafkaConfig {
kafka_hosts: self.kafka_hosts.clone(),
kafka_tls: self.kafka_tls,
kafka_client_id: "usage-ingestion".to_string(),
kafka_compression_codec: self.kafka_compression_codec.clone(),
kafka_producer_linger_ms: self.kafka_producer_linger_ms,
kafka_producer_batch_size: Some(BATCH_SIZE),
kafka_producer_batch_num_messages: Some(BATCH_NUM_MESSAGES),
kafka_producer_message_max_bytes: Some(MESSAGE_MAX_BYTES),
kafka_producer_max_in_flight_requests_per_connection: Some(
MAX_IN_FLIGHT_PER_CONNECTION,
),
kafka_producer_topic_metadata_refresh_interval_ms: Some(METADATA_REFRESH_INTERVAL_MS),
// WarpStream serves a different agent per connection, which collapses throughput
// when librdkafka pins an idempotent producer's sequence numbers to one.
kafka_producer_enable_idempotence: Some(false),
// acks and the partitioner stay at librdkafka's defaults, which match WarpStream's
// advice: acks=all keeps a billing record durable, and these records carry no key.
..Default::default()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn config() -> Config {
Config {
grpc_address: "0.0.0.0:7143".to_string(),
metrics_address: "0.0.0.0:7144".to_string(),
database_url: "postgres://localhost/test".to_string(),
kafka_hosts: "localhost:9092".to_string(),
kafka_tls: false,
kafka_compression_codec: "lz4".to_string(),
kafka_producer_linger_ms: 100,
max_batch_size: 500,
redis_url: String::new(),
redis_flush_interval_seconds: 15,
redis_flush_concurrency: 16,
topic: "clickhouse_billing_usage_records".to_string(),
grpc_max_connection_age_secs: 60,
}
}
#[test]
fn the_producer_follows_the_warpstream_recommendations() {
let kafka = config().kafka_config();
assert_eq!(kafka.kafka_compression_codec, "lz4");
assert_eq!(kafka.kafka_producer_linger_ms, 100);
assert_eq!(kafka.kafka_producer_batch_size, Some(BATCH_SIZE));
assert_eq!(
kafka.kafka_producer_batch_num_messages,
Some(BATCH_NUM_MESSAGES)
);
assert_eq!(
kafka.kafka_producer_message_max_bytes,
Some(MESSAGE_MAX_BYTES)
);
assert_eq!(
kafka.kafka_producer_max_in_flight_requests_per_connection,
Some(MAX_IN_FLIGHT_PER_CONNECTION)
);
assert_eq!(
kafka.kafka_producer_topic_metadata_refresh_interval_ms,
Some(METADATA_REFRESH_INTERVAL_MS)
);
assert_eq!(kafka.kafka_producer_enable_idempotence, Some(false));
}
#[test]
fn the_environment_overrides_compression_and_linger() {
let config = Config {
kafka_compression_codec: "none".to_string(),
kafka_producer_linger_ms: 5,
..config()
};
let kafka = config.kafka_config();
assert_eq!(kafka.kafka_compression_codec, "none");
assert_eq!(kafka.kafka_producer_linger_ms, 5);
}
#[test]
fn redis_counter_configuration_requires_positive_values() {
let config = Config {
redis_flush_concurrency: 0,
..config()
};
assert_eq!(
config.validate(),
Err("USAGE_INGESTION_REDIS_FLUSH_CONCURRENCY must be positive".to_string())
);
}
}