:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use common_types::CapturedEventHeaders;
use futures::channel::oneshot::Canceled;
use metrics::{counter, histogram};
use rdkafka::error::{KafkaError, RDKafkaErrorCode};
use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord, Producer};
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use tracing::error;
use crate::api::CaptureError;
use crate::prometheus::report_dropped_events;
/// A record to be produced to Kafka
#[derive(Debug, Clone)]
pub struct ProduceRecord {
pub topic: Arc<str>,
pub key: Option<String>,
pub payload: bytes::Bytes,
pub headers: CapturedEventHeaders,
}
/// Abstraction over Kafka producer for testability
pub trait KafkaProducer: Send + Sync {
/// The future type returned by send() that resolves to the delivery acknowledgment
type AckFuture: Future<Output = Result<(), CaptureError>> + Send;
/// Send a record to Kafka. Returns either an immediate error or a future to await for ack.
fn send(&self, record: ProduceRecord) -> Result<Self::AckFuture, CaptureError>;
/// Flush pending messages
fn flush(&self) -> Result<(), KafkaError>;
}
/// Future that wraps rdkafka's DeliveryFuture and converts the result to CaptureError.
/// Also records the full app-side ack duration (from `send_result()` returning to
/// broker ack / error / cancellation) as a histogram so we can see the true long
/// tail that the per-broker rdkafka rtt gauge smears away.
///
/// `recorded` flips to true once `poll` observes `Poll::Ready` so the Drop impl
/// can emit a `capture_kafka_produce_ack_duration_ms{outcome="dropped"}` sample
/// when the future is cancelled mid-flight (e.g. `JoinSet::abort_all()` fires
/// after a peer ack fails). Without this, the slowest in-flight acks get
/// censored out of the histogram precisely when the tail matters most.
pub struct DeliveryAckFuture {
inner: DeliveryFuture,
started: Instant,
topic: Arc<str>,
recorded: bool,
}
impl Future for DeliveryAckFuture {
type Output = Result<(), CaptureError>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match Pin::new(&mut self.inner).poll(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(result) => {
// Mark before recording so a panic inside the match still disables Drop.
self.recorded = true;
let elapsed_ms = self.started.elapsed().as_secs_f64() * 1000.0;
let (outcome, mapped) = ack_outcome(
result.map(|delivery| delivery.map(|_| ()).map_err(|(err, _)| err)),
);
histogram!(
"capture_kafka_produce_ack_duration_ms",
"outcome" => outcome,
"topic" => self.topic.clone()
)
.record(elapsed_ms);
Poll::Ready(mapped)
}
}
}
}
fn ack_outcome(
result: Result<Result<(), KafkaError>, Canceled>,
) -> (&'static str, Result<(), CaptureError>) {
match result {
Err(Canceled) => {
// The producer was dropped before the delivery report arrived, so
// the record may or may not have been written.
metrics::counter!("capture_kafka_produce_errors_total").increment(1);
error!("Kafka producer dropped before the delivery report arrived");
("cancelled", Err(CaptureError::RetryableSinkError))
}
Ok(Err(KafkaError::MessageProduction(RDKafkaErrorCode::MessageSizeTooLarge))) => {
report_dropped_events("kafka_message_size", 1);
(
"too_large",
Err(CaptureError::EventTooBig(
"Event rejected by kafka broker during ack".to_string(),
)),
)
}
Ok(Err(err)) => {
metrics::counter!("capture_kafka_produce_errors_total").increment(1);
error!("failed to produce to Kafka: {err:#}");
("broker_err", Err(CaptureError::RetryableSinkError))
}
Ok(Ok(())) => {
metrics::counter!("capture_events_ingested_total").increment(1);
("ok", Ok(()))
}
}
}
fn send_error(err: &KafkaError) -> CaptureError {
match err.rdkafka_error_code() {
Some(RDKafkaErrorCode::MessageSizeTooLarge) => {
report_dropped_events("kafka_message_size", 1);
CaptureError::EventTooBig("Event rejected by kafka during send".to_string())
}
_ => {
// Use error counter, not dropped counter - this is retryable
counter!("capture_kafka_produce_errors_total").increment(1);
error!("failed to produce event: {err:#}");
CaptureError::RetryableSinkError
}
}
}
impl Drop for DeliveryAckFuture {
fn drop(&mut self) {
// Only fires when the future is dropped before poll saw Ready
// (e.g. JoinSet::abort_all on batch fail-fast). Records the elapsed
// wait so tail-latency dashboards don't lose the slowest samples.
if !self.recorded {
let elapsed_ms = self.started.elapsed().as_secs_f64() * 1000.0;
histogram!(
"capture_kafka_produce_ack_duration_ms",
"outcome" => "dropped",
"topic" => self.topic.clone()
)
.record(elapsed_ms);
}
}
}
/// Real Kafka producer implementation using rdkafka
pub struct RdKafkaProducer<C: rdkafka::ClientContext + Send + Sync + 'static> {
producer: FutureProducer<C>,
}
impl<C: rdkafka::ClientContext + Send + Sync + 'static> RdKafkaProducer<C> {
pub fn new(producer: FutureProducer<C>) -> Self {
Self { producer }
}
}
impl<C: rdkafka::ClientContext + Send + Sync + 'static> KafkaProducer for RdKafkaProducer<C> {
type AckFuture = DeliveryAckFuture;
fn send(&self, record: ProduceRecord) -> Result<Self::AckFuture, CaptureError> {
let headers: rdkafka::message::OwnedHeaders = record.headers.into();
let topic = Arc::clone(&record.topic);
match self.producer.send_result(FutureRecord {
topic: &record.topic,
payload: Some(record.payload.as_ref()),
partition: None,
key: record.key.as_deref(),
timestamp: None,
headers: Some(headers),
}) {
Ok(delivery_future) => Ok(DeliveryAckFuture {
inner: delivery_future,
started: Instant::now(),
topic,
recorded: false,
}),
Err((e, _)) => Err(send_error(&e)),
}
}
fn flush(&self) -> Result<(), KafkaError> {
self.producer.flush(Duration::new(30, 0))
}
}
/// Mock Kafka producer for testing - captures all sent records.
///
/// Two failure modes, both keyed on the 0-based send index:
/// - `new_failing_at`: the `idx`-th `send()` itself returns
/// `CaptureError::RetryableSinkError` (an enqueue failure). Records before
/// the failing index are still captured; the failing record is not.
/// - `new_failing_ack_at`: every `send()` succeeds and captures its record,
/// but the `idx`-th ack future resolves to `RetryableSinkError` — the
/// producer accepted the record and its delivery report failed.
#[derive(Clone, Default)]
pub struct MockKafkaProducer {
records: std::sync::Arc<std::sync::Mutex<Vec<ProduceRecord>>>,
fail_at_index: Option<usize>,
fail_ack_at_index: Option<usize>,
call_count: std::sync::Arc<std::sync::atomic::AtomicUsize>,
}
impl MockKafkaProducer {
pub fn new() -> Self {
Self::default()
}
/// Build a producer that returns `RetryableSinkError` on the `idx`-th
/// `send()` call (0-based). All other calls succeed and capture the record.
pub fn new_failing_at(idx: usize) -> Self {
Self {
fail_at_index: Some(idx),
..Self::default()
}
}
/// Build a producer whose `idx`-th (0-based) ack future resolves to
/// `RetryableSinkError` after the send itself succeeded.
pub fn new_failing_ack_at(idx: usize) -> Self {
Self {
fail_ack_at_index: Some(idx),
..Self::default()
}
}
/// Get all records that were sent
pub fn get_records(&self) -> Vec<ProduceRecord> {
self.records.lock().unwrap().clone()
}
/// Clear all captured records
pub fn clear(&self) {
self.records.lock().unwrap().clear();
}
}
impl KafkaProducer for MockKafkaProducer {
type AckFuture = std::future::Ready<Result<(), CaptureError>>;
fn send(&self, record: ProduceRecord) -> Result<Self::AckFuture, CaptureError> {
let idx = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if self.fail_at_index == Some(idx) {
return Err(CaptureError::RetryableSinkError);
}
self.records.lock().unwrap().push(record);
if self.fail_ack_at_index == Some(idx) {
return Ok(std::future::ready(Err(CaptureError::RetryableSinkError)));
}
Ok(std::future::ready(Ok(())))
}
fn flush(&self) -> Result<(), KafkaError> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cancelled_ack_is_retriable() {
let (outcome, mapped) = ack_outcome(Err(Canceled));
assert_eq!(outcome, "cancelled");
assert!(
matches!(mapped, Err(CaptureError::RetryableSinkError)),
"{mapped:?}"
);
}
#[test]
fn queue_full_send_is_retriable() {
let err = KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull);
let mapped = send_error(&err);
assert!(
matches!(mapped, CaptureError::RetryableSinkError),
"{mapped:?}"
);
}
}