:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use envconfig::Envconfig;
#[derive(Envconfig, Clone)]
pub struct KafkaConfig {
#[envconfig(default = "20")]
pub kafka_producer_linger_ms: u32, // Maximum time between producer batches during low traffic
#[envconfig(default = "400")]
pub kafka_producer_queue_mib: u32, // Size of the in-memory producer queue in mebibytes
#[envconfig(default = "10000000")]
pub kafka_producer_queue_messages: u32, // Maximum number of messages in the in-memory producer queue
#[envconfig(default = "20000")]
pub kafka_message_timeout_ms: u32, // Time before we stop retrying producing a message: 20 seconds
#[envconfig(default = "none")]
pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd
#[envconfig(default = "false")]
pub kafka_tls: bool,
#[envconfig(default = "localhost:9092")]
pub kafka_hosts: String,
#[envconfig(default = "")]
pub kafka_client_rack: String,
#[envconfig(default = "")]
pub kafka_client_id: String,
// WarpStream producer tuning — None means "let librdkafka pick its default".
// Reference: https://docs.warpstream.com/warpstream/kafka/configure-kafka-client/tuning-for-performance
pub kafka_producer_batch_size: Option<u32>,
pub kafka_producer_batch_num_messages: Option<u32>,
pub kafka_producer_enable_idempotence: Option<bool>,
pub kafka_producer_max_in_flight_requests_per_connection: Option<u32>,
pub kafka_producer_topic_metadata_refresh_interval_ms: Option<u32>,
pub kafka_producer_message_max_bytes: Option<u32>,
pub kafka_producer_sticky_partitioning_linger_ms: Option<u32>,
// Set to "murmur2_random" to co-partition a keyed topic with Kafka-Java/python-kafka/Node
// producers; `None` uses librdkafka's CRC32-based default, which routes keys differently.
pub kafka_producer_partitioner: Option<String>,
// `None` means "let librdkafka pick its default" (acks=all, effectively unbounded retries
// within message.timeout.ms) — existing callers are unaffected. Fire-and-forget producers
// (e.g. best-effort warning emitters) set these explicitly to acks="1"/retries=0 so a slow
// broker times out and drops instead of retrying.
pub kafka_producer_acks: Option<String>,
pub kafka_producer_retries: Option<u32>,
}
impl KafkaConfig {
pub fn with_endpoint(&self, hosts: &str, tls: bool) -> Self {
Self {
kafka_hosts: hosts.to_string(),
kafka_tls: tls,
..self.clone()
}
}
}
// Keep these values in sync with the `#[envconfig(default = ...)]` attributes
// above: `KafkaConfig::default()` must match `init_from_env()` with no env set,
// so callers that build a config with `..Default::default()` get the same base
// as env-driven callers.
impl Default for KafkaConfig {
fn default() -> Self {
Self {
kafka_producer_linger_ms: 20,
kafka_producer_queue_mib: 400,
kafka_producer_queue_messages: 10_000_000,
kafka_message_timeout_ms: 20_000,
kafka_compression_codec: "none".to_string(),
kafka_tls: false,
kafka_hosts: "localhost:9092".to_string(),
kafka_client_rack: String::new(),
kafka_client_id: String::new(),
kafka_producer_batch_size: None,
kafka_producer_batch_num_messages: None,
kafka_producer_enable_idempotence: None,
kafka_producer_max_in_flight_requests_per_connection: None,
kafka_producer_topic_metadata_refresh_interval_ms: None,
kafka_producer_message_max_bytes: None,
kafka_producer_sticky_partitioning_linger_ms: None,
kafka_producer_partitioner: None,
kafka_producer_acks: None,
kafka_producer_retries: None,
}
}
}
#[derive(Envconfig, Clone)]
pub struct ConsumerConfig {
pub kafka_consumer_group: String,
pub kafka_consumer_topic: String,
// We default to "earliest" for this, but if you're bringing up a new service, you probably want "latest"
#[envconfig(default = "earliest")]
pub kafka_consumer_offset_reset: String, // earliest, latest
// Note: consumers used in a transactional fashion should disable auto offset commits,
// as their offsets should be committed via the transactional producer. All consumers
// disable auto offset /storing/.
pub kafka_consumer_auto_commit: bool,
// expose override config for interval (in milliseconds) between
// Kafka offset commit attempts
#[envconfig(default = "5000")]
pub kafka_consumer_auto_commit_interval_ms: i32,
// Fetch tuning — None means "use librdkafka default" (i.e. don't override).
// WarpStream reads from object storage and benefits from larger, less frequent fetches.
pub kafka_consumer_fetch_wait_max_ms: Option<u32>,
pub kafka_consumer_fetch_min_bytes: Option<u32>,
pub kafka_consumer_fetch_max_bytes: Option<u32>,
pub kafka_consumer_max_partition_fetch_bytes: Option<u32>,
// Static group membership (KIP-345): a stable identity lets a member leave and
// rejoin within session.timeout.ms without a rebalance.
//
// Only set this where the identity survives a restart, e.g. a StatefulSet
// ordinal. A Deployment pod name does not: it changes on every rollout, so the
// rebalance happens anyway. librdkafka also sends no LeaveGroup for a static
// member that stops, so its partitions stay assigned until the session expires.
pub kafka_consumer_group_instance_id: Option<String>,
// Override partition assignment strategy, e.g. "cooperative-sticky" for
// incremental rebalancing instead of the default eager "range,roundrobin".
//
// Name exactly one protocol family. librdkafka rejects a list that mixes eager
// and cooperative strategies, and consumer creation fails. The Java client's
// two-phase "range,cooperative-sticky" upgrade does not work here.
//
// Migrating a live group is one hard switch. Old and new members share no
// common protocol, so the group consumes nothing until the last old member
// leaves. librdkafka retries the join instead of failing, so it recovers on its
// own. Expect a lag spike. A group shared with another service cannot migrate
// until that service moves too.
pub kafka_consumer_partition_strategy: Option<String>,
// WarpStream recommends "0" so the kernel auto-tunes TCP buffers.
pub kafka_consumer_socket_send_buffer_bytes: Option<String>,
pub kafka_consumer_socket_receive_buffer_bytes: Option<String>,
// WarpStream recommends 60000 for faster Agent scaling responsiveness.
pub kafka_consumer_metadata_refresh_interval_ms: Option<u32>,
}
impl ConsumerConfig {
/// Because the consumer config is so application specific, we
/// can't set good defaults in the derive macro, so we expose a way
/// for users to set them here before init'ing their main config struct
pub fn set_defaults(consumer_group: &str, consumer_topic: &str, auto_commit: bool) {
if std::env::var("KAFKA_CONSUMER_GROUP").is_err() {
std::env::set_var("KAFKA_CONSUMER_GROUP", consumer_group);
};
if std::env::var("KAFKA_CONSUMER_TOPIC").is_err() {
std::env::set_var("KAFKA_CONSUMER_TOPIC", consumer_topic);
};
if std::env::var("KAFKA_CONSUMER_AUTO_COMMIT").is_err() {
std::env::set_var("KAFKA_CONSUMER_AUTO_COMMIT", auto_commit.to_string());
}
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use envconfig::Envconfig;
use super::KafkaConfig;
// Guards the two-spot-defaults hazard: `Default` must stay byte-identical to
// the `#[envconfig(default = ...)]` values, so `..Default::default()`
// callers get the same base as env-driven ones. Uses an empty hashmap
// (not process env) to stay deterministic under any CI environment.
#[test]
fn default_matches_envconfig_defaults() {
let d = KafkaConfig::default();
let e = KafkaConfig::init_from_hashmap(&HashMap::new())
.expect("envconfig must initialize from its declared defaults");
assert_eq!(d.kafka_producer_linger_ms, e.kafka_producer_linger_ms);
assert_eq!(d.kafka_producer_queue_mib, e.kafka_producer_queue_mib);
assert_eq!(
d.kafka_producer_queue_messages,
e.kafka_producer_queue_messages
);
assert_eq!(d.kafka_message_timeout_ms, e.kafka_message_timeout_ms);
assert_eq!(d.kafka_compression_codec, e.kafka_compression_codec);
assert_eq!(d.kafka_tls, e.kafka_tls);
assert_eq!(d.kafka_hosts, e.kafka_hosts);
assert_eq!(d.kafka_client_rack, e.kafka_client_rack);
assert_eq!(d.kafka_client_id, e.kafka_client_id);
assert_eq!(d.kafka_producer_batch_size, e.kafka_producer_batch_size);
assert_eq!(
d.kafka_producer_batch_num_messages,
e.kafka_producer_batch_num_messages
);
assert_eq!(
d.kafka_producer_enable_idempotence,
e.kafka_producer_enable_idempotence
);
assert_eq!(
d.kafka_producer_max_in_flight_requests_per_connection,
e.kafka_producer_max_in_flight_requests_per_connection
);
assert_eq!(
d.kafka_producer_topic_metadata_refresh_interval_ms,
e.kafka_producer_topic_metadata_refresh_interval_ms
);
assert_eq!(
d.kafka_producer_message_max_bytes,
e.kafka_producer_message_max_bytes
);
assert_eq!(
d.kafka_producer_sticky_partitioning_linger_ms,
e.kafka_producer_sticky_partitioning_linger_ms
);
assert_eq!(d.kafka_producer_partitioner, e.kafka_producer_partitioner);
assert_eq!(d.kafka_producer_acks, e.kafka_producer_acks);
assert_eq!(d.kafka_producer_retries, e.kafka_producer_retries);
}
}