Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.8 KB
Raw Blame
//! Named Kafka producers, declared in code and instantiated once.
//!
//! A [`ProducerName`] is a connection slot, not a fixed cluster: charts wire
//! each slot to a cluster per deployment, the same model as Node.js
//! ingestion's `KafkaProducerRegistry`. A slot's settings are read from
//! `KAFKA_<SLOT>_PRODUCER_<RDKAFKA_KEY>` variables, and every output that
//! publishes through a slot shares its one producer.

use std::collections::HashMap;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;

use envconfig::Envconfig;
use rdkafka::producer::{FutureProducer, Producer};
use rdkafka::util::Timeout;
use rdkafka::ClientConfig;
use tracing::log::{debug, info};

use crate::sinks::kafka::KafkaContext;
use crate::sinks::producer::{KafkaProducer, RdKafkaProducer};

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ProducerName {
    /// The cluster the ingestion pipelines consume capture's topics from.
    Ingestion,
}

impl ProducerName {
    pub const ALL: [ProducerName; 1] = [ProducerName::Ingestion];

    pub fn as_str(&self) -> &'static str {
        match self {
            ProducerName::Ingestion => "INGESTION",
        }
    }

    // Slot names are single words, so no prefix is a prefix of another.
    fn env_prefix(&self) -> String {
        format!("KAFKA_{}_PRODUCER_", self.as_str())
    }
}

impl FromStr for ProducerName {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        Self::ALL
            .into_iter()
            .find(|name| name.as_str() == s)
            .ok_or_else(|| {
                let known: Vec<&str> = Self::ALL.iter().map(|name| name.as_str()).collect();
                anyhow::anyhow!("unknown producer {s:?}, expected one of {known:?}")
            })
    }
}

/// One producer's connection and tuning. Each field maps to the rdkafka key of
/// the same name.
#[derive(Envconfig, Clone, Debug)]
pub struct ProducerConfig {
    // `kafka:9092` is the broker in local dev and hobby. A deployment that
    // forgets to wire the slot fails its liveness check instead of producing
    // somewhere unexpected.
    #[envconfig(default = "kafka:9092")]
    pub metadata_broker_list: String,
    #[envconfig(default = "")]
    pub security_protocol: String,
    #[envconfig(default = "")]
    pub enable_ssl_certificate_verification: String,
    #[envconfig(default = "")]
    pub client_id: String,
    #[envconfig(default = "none")]
    pub compression_codec: String,
    #[envconfig(default = "20")]
    pub linger_ms: u32,
    #[envconfig(default = "1000000")]
    pub batch_size: u32,
    #[envconfig(default = "10000")]
    pub batch_num_messages: u32,
    #[envconfig(default = "100000")]
    pub queue_buffering_max_messages: u32,
    #[envconfig(default = "409600")]
    pub queue_buffering_max_kbytes: u32,
    #[envconfig(default = "1000000")]
    pub message_max_bytes: u32,
    #[envconfig(default = "20000")]
    pub message_timeout_ms: u32,
    #[envconfig(default = "10")]
    pub sticky_partitioning_linger_ms: u32,
    #[envconfig(default = "20000")]
    pub topic_metadata_refresh_interval_ms: u32,
    #[envconfig(default = "60000")]
    pub metadata_max_age_ms: u32,
    /// `message.send.max.retries`, named `RETRIES` as in Node.js ingestion.
    #[envconfig(default = "2")]
    pub retries: u32,
    #[envconfig(default = "1000000")]
    pub max_in_flight_requests_per_connection: u32,
    #[envconfig(default = "false")]
    pub enable_idempotence: bool,
    #[envconfig(default = "all")]
    pub acks: String,
    #[envconfig(default = "murmur2_random")]
    pub partitioner: String,
    #[envconfig(default = "60000")]
    pub socket_timeout_ms: u32,
    #[envconfig(default = "")]
    pub broker_address_family: String,
    #[envconfig(default = "true")]
    pub log_connection_close: bool,
    #[envconfig(default = "1000")]
    pub retry_backoff_max_ms: u32,
    #[envconfig(default = "0")]
    pub socket_send_buffer_bytes: u32,
    #[envconfig(default = "0")]
    pub socket_receive_buffer_bytes: u32,
}

impl ProducerConfig {
    pub fn load(name: ProducerName, env: &HashMap<String, String>) -> anyhow::Result<Self> {
        let prefix = name.env_prefix();
        let slot_env: HashMap<String, String> = env
            .iter()
            .filter_map(|(k, v)| {
                k.strip_prefix(&prefix)
                    .map(|key| (key.to_string(), v.clone()))
            })
            .collect();
        Self::init_from_hashmap(&slot_env)
            .map_err(|e| anyhow::anyhow!("producer {}: {e}", name.as_str()))
    }

    fn client_config(&self) -> ClientConfig {
        let mut client_config = ClientConfig::new();
        client_config
            .set("bootstrap.servers", &self.metadata_broker_list)
            .set("statistics.interval.ms", "10000")
            .set("partitioner", &self.partitioner)
            .set("metadata.max.age.ms", self.metadata_max_age_ms.to_string())
            .set(
                "topic.metadata.refresh.interval.ms",
                self.topic_metadata_refresh_interval_ms.to_string(),
            )
            .set("message.send.max.retries", self.retries.to_string())
            .set("linger.ms", self.linger_ms.to_string())
            .set("message.max.bytes", self.message_max_bytes.to_string())
            .set("message.timeout.ms", self.message_timeout_ms.to_string())
            .set("socket.timeout.ms", self.socket_timeout_ms.to_string())
            .set("compression.codec", &self.compression_codec)
            .set(
                "queue.buffering.max.kbytes",
                self.queue_buffering_max_kbytes.to_string(),
            )
            .set("acks", &self.acks)
            .set("batch.num.messages", self.batch_num_messages.to_string())
            .set("batch.size", self.batch_size.to_string())
            .set(
                "max.in.flight.requests.per.connection",
                self.max_in_flight_requests_per_connection.to_string(),
            )
            .set(
                "sticky.partitioning.linger.ms",
                self.sticky_partitioning_linger_ms.to_string(),
            )
            .set("enable.idempotence", self.enable_idempotence.to_string())
            .set(
                "log.connection.close",
                self.log_connection_close.to_string(),
            )
            .set(
                "queue.buffering.max.messages",
                self.queue_buffering_max_messages.to_string(),
            )
            .set(
                "retry.backoff.max.ms",
                self.retry_backoff_max_ms.to_string(),
            )
            .set(
                "socket.send.buffer.bytes",
                self.socket_send_buffer_bytes.to_string(),
            )
            .set(
                "socket.receive.buffer.bytes",
                self.socket_receive_buffer_bytes.to_string(),
            );

        for (key, value) in [
            ("broker.address.family", &self.broker_address_family),
            ("client.id", &self.client_id),
            ("security.protocol", &self.security_protocol),
            (
                "enable.ssl.certificate.verification",
                &self.enable_ssl_certificate_verification,
            ),
        ] {
            if !value.is_empty() {
                client_config.set(key, value);
            }
        }

        client_config
    }
}

pub type ProducerHandle = Arc<RdKafkaProducer<KafkaContext>>;

/// Every declared producer, each instantiated once.
pub struct ProducerRegistry {
    producers: HashMap<ProducerName, ProducerHandle>,
}

impl ProducerRegistry {
    pub fn build(
        configs: &HashMap<ProducerName, ProducerConfig>,
        mut liveness: HashMap<ProducerName, lifecycle::Handle>,
    ) -> anyhow::Result<Self> {
        let mut producers = HashMap::new();
        for name in ProducerName::ALL {
            let config = configs
                .get(&name)
                .ok_or_else(|| anyhow::anyhow!("no config for producer {}", name.as_str()))?;
            let producer = create_producer(name, config, liveness.remove(&name))?;
            producers.insert(name, producer);
        }
        Ok(Self { producers })
    }

    pub fn get(&self, name: ProducerName) -> ProducerHandle {
        Arc::clone(&self.producers[&name])
    }

    /// Blocks until every producer's queue drains or times out.
    pub fn flush(&self) -> anyhow::Result<()> {
        for producer in self.producers.values() {
            producer.flush()?;
        }
        Ok(())
    }
}

pub fn load_all(
    env: &HashMap<String, String>,
) -> anyhow::Result<HashMap<ProducerName, ProducerConfig>> {
    ProducerName::ALL
        .into_iter()
        .map(|name| Ok((name, ProducerConfig::load(name, env)?)))
        .collect()
}

fn create_producer(
    name: ProducerName,
    config: &ProducerConfig,
    liveness: Option<lifecycle::Handle>,
) -> anyhow::Result<ProducerHandle> {
    info!(
        "connecting producer {} to Kafka brokers at {}...",
        name.as_str(),
        config.metadata_broker_list
    );

    let client_config = config.client_config();
    debug!("rdkafka configuration: {client_config:?}");

    let producer: FutureProducer<KafkaContext> =
        client_config.create_with_context(KafkaContext::new(liveness.clone()))?;

    // A failed ping is not an error, because other sinks may still report healthy.
    if producer
        .client()
        .fetch_metadata(
            Some("__consumer_offsets"),
            Timeout::After(Duration::new(10, 0)),
        )
        .is_ok()
    {
        if let Some(liveness) = &liveness {
            liveness.report_healthy();
        }
        info!("connected to Kafka brokers");
    };

    Ok(Arc::new(RdKafkaProducer::new(producer)))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn env(pairs: &[(&str, &str)]) -> HashMap<String, String> {
        pairs
            .iter()
            .map(|(k, v)| (k.to_string(), v.to_string()))
            .collect()
    }

    #[test]
    fn default_config_sets_the_legacy_rdkafka_settings() {
        let client_config = ProducerConfig::load(ProducerName::Ingestion, &HashMap::new())
            .unwrap()
            .client_config();

        let expected = [
            ("bootstrap.servers", "kafka:9092"),
            ("statistics.interval.ms", "10000"),
            ("partitioner", "murmur2_random"),
            ("metadata.max.age.ms", "60000"),
            ("topic.metadata.refresh.interval.ms", "20000"),
            ("message.send.max.retries", "2"),
            ("linger.ms", "20"),
            ("message.max.bytes", "1000000"),
            ("message.timeout.ms", "20000"),
            ("socket.timeout.ms", "60000"),
            ("compression.codec", "none"),
            ("queue.buffering.max.kbytes", "409600"),
            ("acks", "all"),
            ("batch.num.messages", "10000"),
            ("batch.size", "1000000"),
            ("max.in.flight.requests.per.connection", "1000000"),
            ("sticky.partitioning.linger.ms", "10"),
            ("enable.idempotence", "false"),
            ("log.connection.close", "true"),
            ("queue.buffering.max.messages", "100000"),
            ("retry.backoff.max.ms", "1000"),
            ("socket.send.buffer.bytes", "0"),
            ("socket.receive.buffer.bytes", "0"),
        ];
        for (key, value) in expected {
            assert_eq!(client_config.get(key), Some(value), "{key}");
        }
        for unset in [
            "broker.address.family",
            "client.id",
            "security.protocol",
            "enable.ssl.certificate.verification",
        ] {
            assert_eq!(client_config.get(unset), None, "{unset}");
        }
    }

    #[test]
    fn load_reads_only_its_slot_prefix() {
        let config = ProducerConfig::load(
            ProducerName::Ingestion,
            &env(&[
                ("KAFKA_INGESTION_PRODUCER_METADATA_BROKER_LIST", "msk:9094"),
                ("KAFKA_INGESTION_PRODUCER_SECURITY_PROTOCOL", "ssl"),
                ("KAFKA_INGESTION_PRODUCER_RETRIES", "7"),
                ("KAFKA_HOSTS", "legacy:9092"),
                ("KAFKA_PRODUCER_LINGER_MS", "99"),
            ]),
        )
        .unwrap();

        assert_eq!(config.metadata_broker_list, "msk:9094");
        assert_eq!(config.security_protocol, "ssl");
        assert_eq!(config.retries, 7);
        assert_eq!(config.linger_ms, 20);
    }

    #[test]
    fn load_rejects_an_unparseable_value() {
        let err = ProducerConfig::load(
            ProducerName::Ingestion,
            &env(&[("KAFKA_INGESTION_PRODUCER_LINGER_MS", "soon")]),
        )
        .unwrap_err();

        assert!(err.to_string().contains("INGESTION"), "{err}");
    }

    #[test]
    fn producer_name_parses_only_declared_slots() {
        assert_eq!(
            "INGESTION".parse::<ProducerName>().unwrap(),
            ProducerName::Ingestion
        );
        for unknown in ["ingestion", "WARPSTREAM", ""] {
            let err = unknown.parse::<ProducerName>().unwrap_err();
            assert!(err.to_string().contains("INGESTION"), "{err}");
        }
    }
}