Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.8 KB
Raw Blame
use std::{
    sync::{
        atomic::{AtomicUsize, Ordering},
        Arc,
    },
    time::{Duration, Instant},
};

use anyhow::Error;
use async_trait::async_trait;
use common_kafka::{
    kafka_producer::KafkaProduceError,
    transaction::{KafkaTransaction, TransactionalProducer},
};
use common_types::InternallyCapturedEvent;
use rdkafka::types::RDKafkaErrorCode;
use tracing::{error, info};

use crate::{
    context::AppContext, job::config::KafkaEmitterConfig,
    person_processing_filter::PersonProcessingFilter,
};

use super::{Emitter, Transaction};

pub struct KafkaEmitter {
    producer: TransactionalProducer,
    topic: String,
    send_rate: u64, // Messages sent per second
    person_processing_filter: PersonProcessingFilter,
}

pub struct KafkaEmitterTransaction<'a> {
    inner: KafkaTransaction<'a>,
    topic: &'a str,
    send_rate: u64,
    start: Instant,
    count: AtomicUsize,
    person_processing_filter: &'a PersonProcessingFilter,
}

impl KafkaEmitter {
    pub async fn new(
        emitter_config: KafkaEmitterConfig,
        transactional_id: &str, // Kafka transactional ID
        context: Arc<AppContext>,
    ) -> Result<Self, Error> {
        let producer = TransactionalProducer::from_config(
            &context.config.kafka,
            transactional_id,
            Duration::from_secs(emitter_config.transaction_timeout_seconds),
        )?;

        Ok(Self {
            producer,
            topic: emitter_config.topic,
            send_rate: emitter_config.send_rate,
            person_processing_filter: context.person_processing_filter.clone(),
        })
    }
}

#[async_trait]
impl Emitter for KafkaEmitter {
    fn set_send_rate(&mut self, send_rate: u64) {
        if send_rate != self.send_rate {
            info!(old = self.send_rate, new = send_rate, "send rate changed");
            self.send_rate = send_rate;
        }
    }

    async fn begin_write<'a>(&'a mut self) -> Result<Box<dyn Transaction<'a> + 'a>, Error> {
        let txn = self.producer.begin()?;
        Ok(Box::new(KafkaEmitterTransaction {
            inner: txn,
            start: Instant::now(),
            topic: &self.topic,
            send_rate: self.send_rate,
            count: AtomicUsize::new(0),
            person_processing_filter: &self.person_processing_filter,
        }))
    }
}

#[async_trait]
impl<'a> Transaction<'a> for KafkaEmitterTransaction<'a> {
    async fn emit(&self, data: &[InternallyCapturedEvent]) -> Result<(), Error> {
        for (idx, result) in self
            .inner
            .send_keyed_iter_to_kafka_with_headers(
                self.topic,
                |e| {
                    let disable = self.disable_person_processing(e);
                    self.kafka_key(e, disable)
                },
                |e| {
                    let disable = self.disable_person_processing(e);
                    self.kafka_headers(e, disable)
                },
                data.iter(),
            )
            .await
            .into_iter()
            .enumerate()
        {
            match result {
                Ok(_) => (),
                Err(KafkaProduceError::KafkaProduceError { error })
                    if matches!(
                        error.rdkafka_error_code(),
                        Some(RDKafkaErrorCode::MessageSizeTooLarge)
                    ) =>
                {
                    // We skip these aside from logging them, as there's not much we can do about them
                    error!("Message size too large: {:?}", data[idx].inner);
                }
                Err(err) => return Err(err.into()),
            }
        }

        self.count.fetch_add(data.len(), Ordering::SeqCst);

        Ok(())
    }

    async fn commit_write(self: Box<Self>) -> Result<Duration, Error> {
        let unboxed = *self;
        let count = unboxed.count.load(Ordering::SeqCst);
        let min_duration = unboxed.get_min_txn_duration(count);
        let txn_elapsed = unboxed.start.elapsed();
        let to_sleep = min_duration.saturating_sub(txn_elapsed);
        info!(
            "sent {} messages in {:?}, minimum send duration is {:?}, sleeping for {:?}",
            count, txn_elapsed, min_duration, to_sleep
        );
        unboxed.inner.commit()?;
        info!("committed transaction");
        Ok(to_sleep)
    }

    // Records already produced stay in an open transaction until it ends. Leaving it open holds
    // back read_committed consumers of the topic until the broker times it out.
    async fn abort(self: Box<Self>) -> Result<(), Error> {
        self.inner.abort()?;
        Ok(())
    }
}

impl<'a> KafkaEmitterTransaction<'a> {
    #[inline]
    fn disable_person_processing(&self, e: &InternallyCapturedEvent) -> bool {
        self.person_processing_filter
            .should_disable_person_processing(&e.inner.token, &e.inner.distinct_id)
    }

    #[inline]
    fn kafka_key(
        &self,
        e: &InternallyCapturedEvent,
        disable_person_processing: bool,
    ) -> Option<String> {
        (!disable_person_processing).then(|| e.inner.key())
    }

    #[inline]
    fn kafka_headers(
        &self,
        e: &InternallyCapturedEvent,
        disable_person_processing: bool,
    ) -> Option<rdkafka::message::OwnedHeaders> {
        let mut headers = e.inner.to_headers();
        if disable_person_processing {
            headers.set_force_disable_person_processing(true);
        }
        Some(headers.into())
    }

    fn get_min_txn_duration(&self, txn_count: usize) -> Duration {
        // Get how long the send must take if this is the first send
        let max_send_rate = self.send_rate as f64;
        let batch_size = txn_count as f64;
        Duration::from_secs_f64(batch_size / max_send_rate)
    }
}

#[cfg(test)]
mod tests {
    use common_types::{CapturedEvent, CapturedEventHeaders};
    use uuid::Uuid;

    #[test]
    fn test_captured_event_to_headers() {
        let event = CapturedEvent {
            uuid: Uuid::now_v7(),
            distinct_id: "user123".to_string(),
            session_id: None,
            ip: "127.0.0.1".to_string(),
            data: r#"{"event":"test_event","properties":{}}"#.to_string(),
            now: "2023-10-15T14:30:00+00:00".to_string(),
            sent_at: None,
            token: "test_token".to_string(),
            event: "test_event".to_string(),
            timestamp: chrono::DateTime::parse_from_rfc3339("2023-10-15T14:30:00+00:00")
                .unwrap()
                .with_timezone(&chrono::Utc),
            is_cookieless_mode: false,
            historical_migration: true,
        };

        let headers: CapturedEventHeaders = event.to_headers();

        assert_eq!(headers.token, Some("test_token".to_string()));
        assert_eq!(headers.distinct_id, Some("user123".to_string()));
        assert_eq!(headers.uuid, Some(event.uuid.to_string()));
        assert_eq!(headers.now, Some("2023-10-15T14:30:00+00:00".to_string()));
        assert_eq!(headers.historical_migration, Some(true));
        assert_eq!(headers.force_disable_person_processing, None);
        assert_eq!(
            headers.timestamp,
            Some(event.timestamp.timestamp_millis().to_string())
        );
        assert_eq!(headers.event, Some("test_event".to_string()));
    }

    #[test]
    fn test_headers_conversion_to_owned_headers() {
        let event = CapturedEvent {
            uuid: Uuid::now_v7(),
            distinct_id: "user456".to_string(),
            session_id: None,
            ip: "127.0.0.1".to_string(),
            data: r#"{"event":"another_event","properties":{}}"#.to_string(),
            now: "2023-10-15T15:00:00+00:00".to_string(),
            sent_at: None,
            token: "another_token".to_string(),
            event: "another_event".to_string(),
            timestamp: chrono::DateTime::parse_from_rfc3339("2023-10-15T15:00:00+00:00")
                .unwrap()
                .with_timezone(&chrono::Utc),
            is_cookieless_mode: false,
            historical_migration: false,
        };

        let headers: CapturedEventHeaders = event.to_headers();
        let owned_headers: rdkafka::message::OwnedHeaders = headers.into();

        // Convert back to verify round-trip
        let parsed_headers = CapturedEventHeaders::from(owned_headers);

        assert_eq!(parsed_headers.token, Some("another_token".to_string()));
        assert_eq!(parsed_headers.distinct_id, Some("user456".to_string()));
        assert_eq!(parsed_headers.uuid, Some(event.uuid.to_string()));
        assert_eq!(
            parsed_headers.now,
            Some("2023-10-15T15:00:00+00:00".to_string())
        );
        // historical_migration=false is omitted from headers (None means false)
        assert_eq!(parsed_headers.historical_migration, None);
        assert_eq!(parsed_headers.force_disable_person_processing, None);
        assert_eq!(parsed_headers.event, Some("another_event".to_string()));
    }
}