Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.0 KB
Raw Blame
use crate::avro_schema::AVRO_SCHEMA;
use crate::log_record::{min_kafka_log_row_timestamp_micros, sum_kafka_log_row_bytes, KafkaLogRow};
use crate::metric_record::{sum_kafka_metric_row_bytes, KafkaMetricRow};
use crate::metrics_avro_schema::METRICS_AVRO_SCHEMA;
use crate::trace_record::KafkaTraceRow;
use crate::traces_avro_schema::TRACES_AVRO_SCHEMA;
use anyhow::anyhow;
use apache_avro::{Codec, Schema, Writer, ZstandardSettings};
use capture::config::KafkaConfig;
use chrono::Utc;
use common_kafka::error::error_code_tag;
use health::HealthHandle;
use metrics::{counter, gauge};
use rdkafka::error::KafkaError;
use rdkafka::message::{Header, OwnedHeaders};
use rdkafka::producer::future_producer::OwnedDeliveryResult;
use rdkafka::producer::{FutureProducer, FutureRecord, Producer};
use rdkafka::util::Timeout;
use rdkafka::ClientConfig;
use std::result::Result::Ok;
use std::sync::Arc;
use std::time::Duration;
use tracing::log::{debug, info};

// The logs consumer verifies this signature against the team it resolves from the token.
pub const INTERNAL_LOGS_KAFKA_HEADER: &str = "posthog-internal";

struct KafkaContext {
    liveness: HealthHandle,
}

impl rdkafka::ClientContext for KafkaContext {
    fn stats(&self, stats: rdkafka::Statistics) {
        // Signal liveness, as the main rdkafka loop is running and calling us
        let brokers_up = stats.brokers.values().any(|broker| broker.state == "UP");
        if brokers_up {
            self.liveness.report_healthy_blocking();
        }

        // Update exported metrics
        gauge!("capture_kafka_callback_queue_depth",).set(stats.replyq as f64);
        gauge!("capture_kafka_producer_queue_depth",).set(stats.msg_cnt as f64);
        gauge!("capture_kafka_producer_queue_depth_limit",).set(stats.msg_max as f64);
        gauge!("capture_kafka_producer_queue_bytes",).set(stats.msg_max as f64);
        gauge!("capture_kafka_producer_queue_bytes_limit",).set(stats.msg_size_max as f64);

        for (topic, stats) in stats.topics {
            gauge!(
                "capture_kafka_produce_avg_batch_size_bytes",
                "topic" => topic.clone()
            )
            .set(stats.batchsize.avg as f64);
            gauge!(
                "capture_kafka_produce_avg_batch_size_events",
                "topic" => topic
            )
            .set(stats.batchcnt.avg as f64);
        }

        for (_, stats) in stats.brokers {
            let id_string = format!("{}", stats.nodeid);
            if let Some(rtt) = stats.rtt {
                gauge!(
                    "capture_kafka_produce_rtt_latency_us",
                    "quantile" => "p50",
                    "broker" => id_string.clone()
                )
                .set(rtt.p50 as f64);
                gauge!(
                    "capture_kafka_produce_rtt_latency_us",
                    "quantile" => "p90",
                    "broker" => id_string.clone()
                )
                .set(rtt.p90 as f64);
                gauge!(
                    "capture_kafka_produce_rtt_latency_us",
                    "quantile" => "p95",
                    "broker" => id_string.clone()
                )
                .set(rtt.p95 as f64);
                gauge!(
                    "capture_kafka_produce_rtt_latency_us",
                    "quantile" => "p99",
                    "broker" => id_string.clone()
                )
                .set(rtt.p99 as f64);
            }

            gauge!(
                "capture_kafka_broker_requests_pending",
                "broker" => id_string.clone()
            )
            .set(stats.outbuf_cnt as f64);
            gauge!(
                "capture_kafka_broker_responses_awaiting",
                "broker" => id_string.clone()
            )
            .set(stats.waitresp_cnt as f64);
            counter!(
                "capture_kafka_broker_tx_errors_total",
                "broker" => id_string.clone()
            )
            .absolute(stats.txerrs);
            counter!(
                "capture_kafka_broker_rx_errors_total",
                "broker" => id_string.clone()
            )
            .absolute(stats.rxerrs);
            counter!(
                "capture_kafka_broker_request_timeouts",
                "broker" => id_string
            )
            .absolute(stats.req_timeouts);
        }
    }
}

#[derive(Clone)]
pub struct KafkaSink {
    logs_producer: FutureProducer<KafkaContext>,
    traces_producer: FutureProducer<KafkaContext>,
    metrics_producer: FutureProducer<KafkaContext>,
    logs_topic: String,
    traces_topic: String,
    metrics_topic: String,
    // Shared, because `KafkaSink` is cloned into every request's state and a `Schema` clone is
    // a few hundred allocations.
    logs_schema: Arc<Schema>,
    traces_schema: Arc<Schema>,
    metrics_schema: Arc<Schema>,
    logs_message_max_bytes: usize,
}

#[allow(clippy::too_many_arguments)]
fn build_client_config(
    bootstrap_servers: &str,
    tls: bool,
    client_id: &str,
    compression_codec: &str,
    producer_acks: &str,
    producer_linger_ms: u32,
    producer_queue_mib: u32,
    message_timeout_ms: u32,
    producer_message_max_bytes: u32,
    producer_max_retries: u32,
    topic_metadata_refresh_interval_ms: u32,
    metadata_max_age_ms: u32,
) -> ClientConfig {
    let mut client_config = ClientConfig::new();
    client_config
        .set("bootstrap.servers", bootstrap_servers)
        .set("statistics.interval.ms", "10000")
        .set("partitioner", "murmur2_random") // Compatibility with python-kafka
        .set("metadata.max.age.ms", metadata_max_age_ms.to_string())
        .set(
            "topic.metadata.refresh.interval.ms",
            topic_metadata_refresh_interval_ms.to_string(),
        )
        .set("message.send.max.retries", producer_max_retries.to_string())
        .set("linger.ms", producer_linger_ms.to_string())
        .set("message.max.bytes", producer_message_max_bytes.to_string())
        .set("message.timeout.ms", message_timeout_ms.to_string())
        .set("compression.codec", compression_codec)
        .set(
            "queue.buffering.max.kbytes",
            (producer_queue_mib * 1024).to_string(),
        )
        .set("acks", producer_acks);

    if !client_id.is_empty() {
        client_config.set("client.id", client_id);
    }

    if tls {
        client_config
            .set("security.protocol", "ssl")
            .set("enable.ssl.certificate.verification", "false");
    }

    client_config
}

async fn build_producer(
    client_config: ClientConfig,
    liveness: HealthHandle,
    label: &str,
) -> anyhow::Result<FutureProducer<KafkaContext>> {
    debug!("rdkafka {label} configuration: {client_config:?}");
    let producer: FutureProducer<KafkaContext> =
        client_config.create_with_context(KafkaContext {
            liveness: liveness.clone(),
        })?;

    // Ping the cluster to make sure we can reach brokers, fail after 10 seconds
    // Note: we don't error if we fail to connect as there may be other sinks that report healthy
    if producer
        .client()
        .fetch_metadata(
            Some("__consumer_offsets"),
            Timeout::After(Duration::new(10, 0)),
        )
        .is_ok()
    {
        liveness.report_healthy().await;
        info!("connected to Kafka brokers ({label})");
    };

    Ok(producer)
}

fn count_produce_error(topic: &str, reason: &'static str) {
    counter!(
        "capture_kafka_produce_errors_total",
        "topic" => Arc::<str>::from(topic),
        "reason" => reason
    )
    .increment(1);
}

/// `None` means the delivery report never arrived, which is what a producer teardown
/// with the batch still in flight looks like. A `message.timeout.ms` expiry is not this
/// case: librdkafka always reports it, as an inner `MessageTimedOut`.
fn interpret_delivery_result(
    result: Option<OwnedDeliveryResult>,
    topic: &str,
) -> Result<(), anyhow::Error> {
    match result {
        None => {
            count_produce_error(topic, "delivery_cancelled");
            Err(anyhow!(
                "kafka error: the producer dropped the batch without a delivery report"
            ))
        }
        Some(Err((err, _))) => {
            count_produce_error(topic, kafka_error_tag(&err));
            Err(anyhow!("kafka error: delivery failed: {err}"))
        }
        Some(Ok(_)) => Ok(()),
    }
}

fn kafka_error_tag(err: &KafkaError) -> &'static str {
    err.rdkafka_error_code()
        .map(error_code_tag)
        .unwrap_or("rdkafka_other")
}

impl KafkaSink {
    pub async fn new(
        config: KafkaConfig,
        logs_liveness: HealthHandle,
        traces_liveness: HealthHandle,
        metrics_liveness: HealthHandle,
    ) -> anyhow::Result<KafkaSink> {
        info!(
            "connecting to logs Kafka brokers at {}...",
            config.kafka_hosts
        );
        let logs_client_config = build_client_config(
            &config.kafka_hosts,
            config.kafka_tls,
            &config.kafka_client_id,
            &config.kafka_compression_codec,
            &config.kafka_producer_acks,
            config.kafka_producer_linger_ms,
            config.kafka_producer_queue_mib,
            config.kafka_message_timeout_ms,
            config.kafka_producer_message_max_bytes,
            config.kafka_producer_max_retries,
            config.kafka_topic_metadata_refresh_interval_ms,
            config.kafka_metadata_max_age_ms,
        );
        let logs_producer = build_producer(logs_client_config, logs_liveness, "logs").await?;

        let traces_hosts = config
            .kafka_traces_hosts
            .clone()
            .unwrap_or_else(|| config.kafka_hosts.clone());
        info!("connecting to traces Kafka brokers at {}...", traces_hosts);
        let traces_client_config = build_client_config(
            &traces_hosts,
            config.kafka_traces_tls.unwrap_or(config.kafka_tls),
            &config
                .kafka_traces_client_id
                .clone()
                .unwrap_or_else(|| config.kafka_client_id.clone()),
            &config
                .kafka_traces_compression_codec
                .clone()
                .unwrap_or_else(|| config.kafka_compression_codec.clone()),
            &config
                .kafka_traces_producer_acks
                .clone()
                .unwrap_or_else(|| config.kafka_producer_acks.clone()),
            config
                .kafka_traces_producer_linger_ms
                .unwrap_or(config.kafka_producer_linger_ms),
            config
                .kafka_traces_producer_queue_mib
                .unwrap_or(config.kafka_producer_queue_mib),
            config
                .kafka_traces_message_timeout_ms
                .unwrap_or(config.kafka_message_timeout_ms),
            config
                .kafka_traces_producer_message_max_bytes
                .unwrap_or(config.kafka_producer_message_max_bytes),
            config
                .kafka_traces_producer_max_retries
                .unwrap_or(config.kafka_producer_max_retries),
            config
                .kafka_traces_topic_metadata_refresh_interval_ms
                .unwrap_or(config.kafka_topic_metadata_refresh_interval_ms),
            config
                .kafka_traces_metadata_max_age_ms
                .unwrap_or(config.kafka_metadata_max_age_ms),
        );
        let traces_producer =
            build_producer(traces_client_config, traces_liveness, "traces").await?;

        let metrics_hosts = config
            .kafka_metrics_hosts
            .clone()
            .unwrap_or_else(|| config.kafka_hosts.clone());
        info!(
            "connecting to metrics Kafka brokers at {}...",
            metrics_hosts
        );
        let metrics_client_config = build_client_config(
            &metrics_hosts,
            config.kafka_metrics_tls.unwrap_or(config.kafka_tls),
            &config
                .kafka_metrics_client_id
                .clone()
                .unwrap_or_else(|| config.kafka_client_id.clone()),
            &config
                .kafka_metrics_compression_codec
                .clone()
                .unwrap_or_else(|| config.kafka_compression_codec.clone()),
            &config
                .kafka_metrics_producer_acks
                .clone()
                .unwrap_or_else(|| config.kafka_producer_acks.clone()),
            config
                .kafka_metrics_producer_linger_ms
                .unwrap_or(config.kafka_producer_linger_ms),
            config
                .kafka_metrics_producer_queue_mib
                .unwrap_or(config.kafka_producer_queue_mib),
            config
                .kafka_metrics_message_timeout_ms
                .unwrap_or(config.kafka_message_timeout_ms),
            config
                .kafka_metrics_producer_message_max_bytes
                .unwrap_or(config.kafka_producer_message_max_bytes),
            config
                .kafka_metrics_producer_max_retries
                .unwrap_or(config.kafka_producer_max_retries),
            config
                .kafka_metrics_topic_metadata_refresh_interval_ms
                .unwrap_or(config.kafka_topic_metadata_refresh_interval_ms),
            config
                .kafka_metrics_metadata_max_age_ms
                .unwrap_or(config.kafka_metadata_max_age_ms),
        );
        let metrics_producer =
            build_producer(metrics_client_config, metrics_liveness, "metrics").await?;

        Ok(KafkaSink {
            logs_producer,
            traces_producer,
            metrics_producer,
            logs_topic: config.kafka_topic,
            traces_topic: config.kafka_traces_topic,
            metrics_topic: config.kafka_metrics_topic,
            logs_schema: Arc::new(Schema::parse_str(AVRO_SCHEMA)?),
            traces_schema: Arc::new(Schema::parse_str(TRACES_AVRO_SCHEMA)?),
            metrics_schema: Arc::new(Schema::parse_str(METRICS_AVRO_SCHEMA)?),
            logs_message_max_bytes: config.kafka_producer_message_max_bytes as usize,
        })
    }

    /// The producer's `message.max.bytes` for the logs topic, so an intake that chunks a large
    /// delivery can size its batches against the real cap instead of a guess.
    pub fn logs_message_max_bytes(&self) -> usize {
        self.logs_message_max_bytes
    }

    pub fn flush(&self) -> Result<(), KafkaError> {
        // TODO: hook it up on shutdown
        self.logs_producer.flush(Duration::new(30, 0))?;
        self.traces_producer.flush(Duration::new(30, 0))?;
        self.metrics_producer.flush(Duration::new(30, 0))
    }

    #[allow(clippy::too_many_arguments)]
    async fn write_avro_batch<T: serde::Serialize>(
        &self,
        producer: &FutureProducer<KafkaContext>,
        topic: &str,
        schema: &Schema,
        token: &str,
        rows: &[T],
        uncompressed_bytes: u64,
        records_uncompressed_bytes: Option<u64>,
        timestamps_overridden: u64,
        min_timestamp_micros: Option<i64>,
        backfill_days: Option<u32>,
        internal_signature: Option<&str>,
    ) -> Result<(), anyhow::Error> {
        let mut writer = Writer::with_codec(
            schema,
            Vec::new(),
            Codec::Zstandard(ZstandardSettings::new(1)),
        );

        for row in rows {
            writer.append_ser(row)?;
        }

        let payload: Vec<u8> = writer.into_inner()?;

        let future = match producer.send_result(FutureRecord {
            topic,
            payload: Some(&payload),
            partition: None,
            key: None::<Vec<u8>>.as_ref(),
            timestamp: None,
            headers: Some({
                let created_at = Utc::now().to_rfc3339();
                let mut headers = OwnedHeaders::new()
                    .insert(Header {
                        key: "token",
                        value: Some(&token.to_string()),
                    })
                    .insert(Header {
                        key: "bytes_uncompressed",
                        value: Some(&uncompressed_bytes.to_string()),
                    });
                // Records-based size next to the payload-based `bytes_uncompressed`, so
                // billing can compare the two before switching to the records-based value.
                if let Some(records_bytes) = records_uncompressed_bytes {
                    headers = headers.insert(Header {
                        key: "bytes_uncompressed_records",
                        value: Some(&records_bytes.to_string()),
                    });
                }
                if let Some(micros) = min_timestamp_micros {
                    headers = headers.insert(Header {
                        key: "min_timestamp",
                        value: Some(&micros.to_string()),
                    });
                }
                // A backfill request can carry recent rows, so row age cannot identify it.
                // The Node consumer reads this header to apply its per-team backfill list.
                if let Some(days) = backfill_days {
                    headers = headers.insert(Header {
                        key: "backfill_days",
                        value: Some(&days.to_string()),
                    });
                }
                if let Some(signature) = internal_signature {
                    headers = headers.insert(Header {
                        key: INTERNAL_LOGS_KAFKA_HEADER,
                        value: Some(signature),
                    });
                }
                headers
                    .insert(Header {
                        key: "bytes_compressed",
                        value: Some(&payload.len().to_string()),
                    })
                    .insert(Header {
                        key: "record_count",
                        value: Some(&rows.len().to_string()),
                    })
                    .insert(Header {
                        key: "created_at",
                        value: Some(&created_at),
                    })
                    .insert(Header {
                        key: "batch_uuid",
                        value: Some(&uuid::Uuid::new_v4().to_string()),
                    })
                    .insert(Header {
                        key: "timestamps_overridden",
                        value: Some(&timestamps_overridden.to_string()),
                    })
            }),
        }) {
            Err((err, _)) => {
                count_produce_error(topic, "enqueue");
                Err(anyhow!("kafka error: {err}"))
            }
            Ok(delivery_future) => Ok(delivery_future),
        }?;

        interpret_delivery_result(future.await.ok(), topic)
    }

    pub async fn write(
        &self,
        token: &str,
        rows: Vec<KafkaLogRow>,
        uncompressed_bytes: u64,
        timestamps_overridden: u64,
        backfill_days: Option<u32>,
        internal_signature: Option<&str>,
    ) -> Result<(), anyhow::Error> {
        if rows.is_empty() {
            return Ok(());
        }

        if timestamps_overridden > 0 {
            counter!("capture_logs_timestamps_overridden").increment(timestamps_overridden);
        }

        let records_uncompressed_bytes = sum_kafka_log_row_bytes(&rows);
        counter!("capture_logs_bytes_uncompressed_payload").increment(uncompressed_bytes);
        counter!("capture_logs_bytes_uncompressed_records").increment(records_uncompressed_bytes);
        if records_uncompressed_bytes > uncompressed_bytes {
            // Records sum should stay below the payload size (it excludes transport overhead);
            // billing can only switch to it if that invariant holds.
            counter!("capture_logs_records_bytes_exceed_payload").increment(1);
        }

        self.write_avro_batch(
            &self.logs_producer,
            &self.logs_topic,
            &self.logs_schema,
            token,
            &rows,
            uncompressed_bytes,
            Some(records_uncompressed_bytes),
            timestamps_overridden,
            min_kafka_log_row_timestamp_micros(&rows),
            backfill_days,
            internal_signature,
        )
        .await?;

        Ok(())
    }

    pub async fn write_traces(
        &self,
        token: &str,
        rows: Vec<KafkaTraceRow>,
        uncompressed_bytes: u64,
        timestamps_overridden: u64,
    ) -> Result<(), anyhow::Error> {
        if rows.is_empty() {
            return Ok(());
        }

        if timestamps_overridden > 0 {
            counter!("capture_traces_timestamps_overridden").increment(timestamps_overridden);
        }

        self.write_avro_batch(
            &self.traces_producer,
            &self.traces_topic,
            &self.traces_schema,
            token,
            &rows,
            uncompressed_bytes,
            None,
            timestamps_overridden,
            None,
            None,
            None,
        )
        .await?;

        Ok(())
    }

    pub async fn write_metrics(
        &self,
        token: &str,
        rows: Vec<KafkaMetricRow>,
        uncompressed_bytes: u64,
        timestamps_overridden: u64,
    ) -> Result<(), anyhow::Error> {
        if rows.is_empty() {
            return Ok(());
        }

        if timestamps_overridden > 0 {
            counter!("capture_metrics_timestamps_overridden").increment(timestamps_overridden);
        }

        let records_uncompressed_bytes = sum_kafka_metric_row_bytes(&rows);
        counter!("capture_metrics_bytes_uncompressed_payload").increment(uncompressed_bytes);
        counter!("capture_metrics_bytes_uncompressed_records")
            .increment(records_uncompressed_bytes);

        self.write_avro_batch(
            &self.metrics_producer,
            &self.metrics_topic,
            &self.metrics_schema,
            token,
            &rows,
            uncompressed_bytes,
            Some(records_uncompressed_bytes),
            timestamps_overridden,
            None,
            None,
            None,
        )
        .await?;

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rdkafka::error::RDKafkaErrorCode;
    use rdkafka::message::OwnedMessage;
    use rdkafka::Timestamp;

    fn rejected_by_broker(code: RDKafkaErrorCode) -> Option<OwnedDeliveryResult> {
        Some(Err((
            KafkaError::MessageProduction(code),
            OwnedMessage::new(
                None,
                None,
                "logs".to_string(),
                Timestamp::NotAvailable,
                0,
                0,
                None,
            ),
        )))
    }

    #[test]
    fn a_failed_delivery_is_never_reported_as_a_write() {
        assert!(interpret_delivery_result(None, "logs").is_err());
        assert!(interpret_delivery_result(
            rejected_by_broker(RDKafkaErrorCode::MessageTimedOut),
            "logs"
        )
        .is_err());
    }

    #[test]
    fn a_delivered_batch_is_reported_as_a_write() {
        assert!(interpret_delivery_result(Some(Ok((0, 42))), "logs").is_ok());
    }

    #[test]
    fn a_broker_error_is_tagged_with_the_shared_vocabulary() {
        assert_eq!(
            kafka_error_tag(&KafkaError::MessageProduction(
                RDKafkaErrorCode::MessageSizeTooLarge
            )),
            "message_size_too_large"
        );
        assert_eq!(
            kafka_error_tag(&KafkaError::NoMessageReceived),
            "rdkafka_other"
        );
    }
}