:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Kafka sink (mechanism layer).
//!
//! This sink is pure mechanism: it produces to Kafka with `rdkafka`.
//! `KafkaSinkBase::prepare_record` serializes a `ProcessedEvent` and maps the
//! metadata the pipeline stamped to a [`Destination`] and partition key; v1's
//! prepared events arrive serialized and carry their own address. Routing
//! policy lives in `pipeline::resolve`, which reads the stamped metadata and
//! the output table's `has_ai_overflow_lane`. Enqueue resolves the destination
//! to a topic and producer.
use crate::api::CaptureError;
use crate::config::EnvelopeCompression;
use crate::ordering::OrderingGuarantee;
use crate::outputs::{PreparedEvent, PublishEvents, PublishPrepared};
use crate::pipeline::{self, Address};
use crate::producers::ProducerHandle;
use crate::serialization::Serializer;
use crate::sinks::producer::{KafkaProducer, ProduceRecord};
use crate::sinks::registry::{Destination, OutputTable};
use crate::sinks::sink::{fold_results, Outcome, PreparedPayload, PublishPayloads, SinkResult};
use crate::v0_request::{DataType, ProcessedEvent};
use async_trait::async_trait;
use bytes::Bytes;
use common_types::CapturedEventHeaders;
use futures::future::join_all;
use metrics::{counter, gauge, histogram};
use std::future::Future;
use std::sync::Arc;
use std::time::Instant;
use tokio::task::JoinSet;
use tracing::log::error;
use tracing::{info_span, instrument, Instrument};
use super::producer::RdKafkaProducer;
pub struct KafkaContext {
/// Lifecycle handle this producer reports liveness to. `None` for a producer
/// whose health must not gate the pod — it still produces and emits
/// metrics, it just doesn't drive a manager component.
liveness: Option<lifecycle::Handle>,
}
impl KafkaContext {
pub(crate) fn new(liveness: Option<lifecycle::Handle>) -> Self {
Self { liveness }
}
}
/// Emit min/avg/max/stddev plus p50/p90/p95/p99 for an rdkafka window stat
/// (rtt, int_latency, outbuf_latency). Gauges are tagged with `quantile` and
/// `broker` so existing dashboards keyed on `quantile` keep working and new
/// panels can pick up `max`/`avg` for tail visibility.
fn emit_window_stats(
metric_name: &'static str,
window: &rdkafka::statistics::Window,
broker: &str,
) {
for (quantile, value) in [
("min", window.min),
("avg", window.avg),
("max", window.max),
("stddev", window.stddev),
("p50", window.p50),
("p90", window.p90),
("p95", window.p95),
("p99", window.p99),
] {
gauge!(
metric_name,
"quantile" => quantile,
"broker" => broker.to_string()
)
.set(value as f64);
}
}
impl rdkafka::ClientContext for KafkaContext {
fn stats(&self, stats: rdkafka::Statistics) {
let brokers_up = stats.brokers.values().any(|broker| broker.state == "UP");
if brokers_up {
if let Some(liveness) = &self.liveness {
liveness.report_healthy();
}
}
let total_brokers = stats.brokers.len();
let up_brokers = stats
.brokers
.values()
.filter(|broker| broker.state == "UP")
.count();
let down_brokers = total_brokers.saturating_sub(up_brokers);
gauge!("capture_kafka_any_brokers_down").set(if down_brokers > 0 { 1.0 } else { 0.0 });
gauge!("capture_kafka_callback_queue_depth",).set(stats.replyq as f64);
gauge!("capture_kafka_producer_queue_depth",).set(stats.msg_cnt as f64);
gauge!("capture_kafka_producer_queue_depth_limit",).set(stats.msg_max as f64);
gauge!("capture_kafka_producer_queue_bytes",).set(stats.msg_size as f64);
gauge!("capture_kafka_producer_queue_bytes_limit",).set(stats.msg_size_max as f64);
for (topic, stats) in stats.topics {
gauge!(
"capture_kafka_produce_avg_batch_size_bytes",
"topic" => topic.clone()
)
.set(stats.batchsize.avg as f64);
gauge!(
"capture_kafka_produce_avg_batch_size_events",
"topic" => topic
)
.set(stats.batchcnt.avg as f64);
}
for (_, stats) in stats.brokers {
let id_string = format!("{}", stats.nodeid);
// Per-broker connectivity (1 = connected/UP, 0 = not connected)
gauge!(
"capture_kafka_broker_connected",
"broker" => id_string.clone()
)
.set(if stats.state == "UP" { 1.0 } else { 0.0 });
if let Some(rtt) = stats.rtt {
emit_window_stats("capture_kafka_produce_rtt_latency_us", &rtt, &id_string);
}
// Time messages spent in the producer's internal queue (linger + backlog).
// Usually the dominant source of long-tail ack delays when brokers are slow.
if let Some(int_latency) = stats.int_latency {
emit_window_stats(
"capture_kafka_produce_int_latency_us",
&int_latency,
&id_string,
);
}
// Time requests spent in the broker's output buffer before going on the wire.
if let Some(outbuf_latency) = stats.outbuf_latency {
emit_window_stats(
"capture_kafka_produce_outbuf_latency_us",
&outbuf_latency,
&id_string,
);
}
gauge!(
"capture_kafka_broker_requests_pending",
"broker" => id_string.clone()
)
.set(stats.outbuf_cnt as f64);
gauge!(
"capture_kafka_broker_responses_awaiting",
"broker" => id_string.clone()
)
.set(stats.waitresp_cnt as f64);
counter!(
"capture_kafka_broker_tx_errors_total",
"broker" => id_string.clone()
)
.absolute(stats.txerrs);
counter!(
"capture_kafka_broker_rx_errors_total",
"broker" => id_string.clone()
)
.absolute(stats.rxerrs);
counter!(
"capture_kafka_broker_request_timeouts",
"broker" => id_string
)
.absolute(stats.req_timeouts);
}
}
}
/// Generic Kafka sink that can use any producer implementation.
pub struct KafkaSinkBase<P: KafkaProducer> {
outputs: Arc<OutputTable<Arc<P>>>,
replay_envelope_compression: EnvelopeCompression,
}
impl<P: KafkaProducer> Clone for KafkaSinkBase<P> {
fn clone(&self) -> Self {
Self {
outputs: Arc::clone(&self.outputs),
replay_envelope_compression: self.replay_envelope_compression,
}
}
}
/// The dlq output's contract: count the reroute and stamp the dlq header set.
fn dlq_reroute_effects(headers: &mut common_types::CapturedEventHeaders, reason: &'static str) {
counter!("capture_events_rerouted_dlq", &[("reason", reason)]).increment(1);
headers.set_dlq_reason(reason.to_string());
// Unlike with our node code, DLQ step will always be static.
headers.set_dlq_step("capture".to_string());
headers
.set_dlq_timestamp(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true));
}
/// The default KafkaSink using rdkafka's FutureProducer
pub type KafkaSink = KafkaSinkBase<RdKafkaProducer<KafkaContext>>;
impl KafkaSink {
pub fn new(
outputs: OutputTable<ProducerHandle>,
replay_envelope_compression: EnvelopeCompression,
) -> KafkaSink {
KafkaSinkBase {
outputs: Arc::new(outputs),
replay_envelope_compression,
}
}
}
impl<P: KafkaProducer> KafkaSinkBase<P> {
/// Create a KafkaSinkBase over a custom producer, for tests.
pub fn with_producer(producer: P, outputs: OutputTable) -> Self {
Self::with_producer_and_compression(producer, outputs, EnvelopeCompression::None)
}
/// Same as `with_producer` but with envelope compression enabled. Used in tests.
pub fn with_producer_and_compression(
producer: P,
outputs: OutputTable,
replay_envelope_compression: EnvelopeCompression,
) -> Self {
let producer = Arc::new(producer);
Self {
outputs: Arc::new(outputs.map_producers(|_| Arc::clone(&producer))),
replay_envelope_compression,
}
}
/// CPU-bound prep work: serialize payload + build headers + pick topic/key.
/// Safe to run concurrently across events in a batch because it does not
/// touch the librdkafka producer queue — `PublishPayloads::publish` is what enforces
/// per-partition ordering by calling `enqueue_record` serially in the
/// original event order.
///
/// Routing policy is read from `ProcessedEventMetadata` (stamped upstream
/// by the pipeline). This function does not consult any limiter — it is
/// pure mechanism. DLQ and custom-topic redirects take priority over
/// overflow routing.
///
/// Not `async`: there are no await points, and keeping it
/// synchronous lets `prepare_batch`'s serial fast path call it inline
/// without any runtime indirection.
fn prepare_record(&self, event: ProcessedEvent) -> Result<PreparedPayload, CaptureError> {
let (event, metadata) = (event.event, event.metadata);
let uuid = event.uuid;
// Encoding is resolved by data_type, not by the routed destination:
// session replay gets the lz4 envelope when configured, everything
// else the plain JSON contract — so a replay event redirected to the
// DLQ or a custom topic keeps the envelope, with its content-encoding
// header travelling along. See `crate::serialization` for the formats,
// the envelope byte layout, and the coexistence story.
let serializer = match (metadata.data_type, self.replay_envelope_compression) {
(DataType::SnapshotMain, EnvelopeCompression::Lz4) => Serializer::JSON_LZ4,
_ => Serializer::JSON,
};
let payload = serializer.serialize(&event).map_err(|e| {
error!(
"failed to serialize {} event payload: {e:#}",
metadata.event_name
);
CaptureError::NonRetryableSinkError
})?;
let event_key = event.key();
let mut headers = event.to_headers();
drop(event); // Events can be EXTREMELY memory hungry
// The stamped flag (event restrictions / upstream decisions) or a
// ForceLimited reason, which implies the skip on its own.
if metadata.person_processing_disabled() {
headers.set_force_disable_person_processing(true);
}
if metadata.skip_heatmap_processing {
headers.set_skip_heatmap_processing(true);
}
// The pipeline layer owns the address decision. Each address's side
// effects live in the same match that maps it to an output, so a new
// address cannot skip them: dlq sets the dlq headers, and both admin
// redirects count their reroutes.
let decision = pipeline::resolve(&metadata, self.outputs.has_ai_overflow_lane())?;
let target = match decision.address {
Address::Dlq => {
dlq_reroute_effects(&mut headers, "event_restriction");
Destination::Dlq
}
Address::Custom(topic) => {
counter!(
"capture_events_rerouted_custom_topic",
&[("reason", "event_restriction")]
)
.increment(1);
Destination::Custom(topic)
}
Address::Lane(lane) => Destination::for_lane(lane),
};
let partition_key = match decision.ordering {
// resolve() rejects replay events without a session id, so the id
// is present whenever PerSession is decided.
OrderingGuarantee::PerSession => {
metadata.session_id.ok_or(CaptureError::MissingSessionId)?
}
OrderingGuarantee::PerDistinctId | OrderingGuarantee::None => event_key,
};
if let Some(encoding) = serializer.content_encoding() {
headers.set_content_encoding(encoding.to_string());
}
Ok(PreparedPayload {
uuid,
destination: target,
partition_key,
ordering: decision.ordering,
payload,
headers,
})
}
/// librdkafka preserves on-wire partition order by `send_result` call
/// order, so this MUST be called in the original event order within a
/// batch.
fn enqueue_record(&self, payload: PreparedPayload) -> Result<P::AckFuture, CaptureError> {
self.send(
&payload.destination,
payload.partition_key,
payload.ordering,
payload.payload.into(),
payload.headers,
)
}
fn send(
&self,
destination: &Destination,
partition_key: String,
ordering: OrderingGuarantee,
payload: Bytes,
headers: CapturedEventHeaders,
) -> Result<P::AckFuture, CaptureError> {
let (topic, producer) = self.outputs.resolve(destination);
counter!("capture_kafka_produce_bytes_total", "topic" => Arc::clone(&topic))
.increment(payload.len() as u64);
// No key reaches rdkafka as round-robin; `Some("")` would murmur2-hash
// every record onto one deterministic hot partition.
let key = match ordering {
OrderingGuarantee::None => None,
_ => Some(partition_key),
};
producer.send(ProduceRecord {
topic,
key,
payload,
headers,
})
}
/// Prep + enqueue for the one-event fast path, returning the raw ack
/// future so a single send can await it inline.
///
/// Records the same two phase histograms as the batch path, on the error
/// exits as well, so a publish of one event stays in the same population
/// as a publish of many. Without this, the cheapest publishes leave the
/// distribution and the in-process quantiles step up on their own.
fn kafka_send(&self, event: ProcessedEvent) -> Result<P::AckFuture, CaptureError> {
let prep_start = Instant::now();
let payload = self.prepare_record(event);
histogram!("capture_kafka_batch_prep_duration_seconds")
.record(prep_start.elapsed().as_secs_f64());
let enqueue_start = Instant::now();
let ack_future = self.enqueue_record(payload?);
histogram!("capture_kafka_batch_enqueue_duration_seconds")
.record(enqueue_start.elapsed().as_secs_f64());
ack_future
}
}
/// Batches below this size take the serial fast path in `prepare_batch`:
/// spawning N `JoinSet` tasks to run `prepare_record` in parallel is
/// net-negative when each task does only a `serde_json::to_string` and a
/// header build — the scheduler overhead dominates the CPU savings.
/// Scatter-gather kicks in at or above this threshold where parallel prep
/// wins back its spawn cost.
pub(crate) const SCATTER_GATHER_MIN_BATCH: usize = 8;
impl<P: KafkaProducer + 'static> KafkaSinkBase<P> {
/// CPU-bound batch prep: run `prepare_record` over the batch and return
/// the payloads in the original event order, fail-fast on the first prep
/// error so no partially-prepped batch reaches the producer. Inherent
/// rather than on the `PublishPayloads` trait: payload assembly is not
/// backend mechanism.
pub(crate) async fn prepare_batch(
&self,
events: Vec<ProcessedEvent>,
) -> Result<Vec<PreparedPayload>, CaptureError> {
let batch_size = events.len();
// Small-batch fast path. For batches under `SCATTER_GATHER_MIN_BATCH`
// the JoinSet spawn overhead dominates any parallel-prep win, so we
// stay single-threaded. Both paths share the semantic "prep error ->
// no records produced": all events prep before anything is enqueued.
// The duration histogram is recorded on every exit so dashboards
// keep a faithful view of the fast path.
if batch_size < SCATTER_GATHER_MIN_BATCH {
let prep_start = Instant::now();
let mut prepared: Vec<PreparedPayload> = Vec::with_capacity(batch_size);
for event in events {
match self.prepare_record(event) {
Ok(payload) => prepared.push(payload),
Err(err) => {
histogram!("capture_kafka_batch_prep_duration_seconds")
.record(prep_start.elapsed().as_secs_f64());
return Err(err);
}
}
}
histogram!("capture_kafka_batch_prep_duration_seconds")
.record(prep_start.elapsed().as_secs_f64());
return Ok(prepared);
}
// Parallel prep across tokio workers. Each task returns its input
// index so we can reassemble results in the original event order
// before the serial enqueue phase. This is where the CPU win lives:
// serde_json::to_string + header build run concurrently on up to N
// worker threads, rather than sequentially on a single task.
let prep_start = Instant::now();
let mut prep_set: JoinSet<(usize, Result<PreparedPayload, CaptureError>)> = JoinSet::new();
for (idx, event) in events.into_iter().enumerate() {
let this = self.clone();
prep_set.spawn(
async move { (idx, this.prepare_record(event)) }
.instrument(info_span!("prepare_record")),
);
}
// Collect into a (idx, payload) Vec and sort rather than indexing into
// a `Vec<Option<PreparedPayload>>`. Encodes the "every slot filled"
// invariant in the type: no `Option`, no unreachable `expect`, no
// N-element `None` preallocation. Our only cancellation source is
// `prep_set.abort_all()` below, invoked only from an already-errored
// branch, so any `JoinError` observed during normal drain implies a
// panic inside `prepare_record` — counted separately so it's alertable.
let mut prepared: Vec<(usize, PreparedPayload)> = Vec::with_capacity(batch_size);
while let Some(join_result) = prep_set.join_next().await {
let (idx, result) = match join_result {
Err(err) => {
counter!("capture_kafka_prep_panic_total").increment(1);
error!("join error while preparing Kafka record: {err:#}");
// Record prep duration on the error path too.
prep_set.abort_all();
histogram!("capture_kafka_batch_prep_duration_seconds")
.record(prep_start.elapsed().as_secs_f64());
return Err(CaptureError::RetryableSinkError);
}
Ok(inner) => inner,
};
match result {
Ok(payload) => prepared.push((idx, payload)),
Err(err) => {
prep_set.abort_all();
histogram!("capture_kafka_batch_prep_duration_seconds")
.record(prep_start.elapsed().as_secs_f64());
return Err(err);
}
}
}
prepared.sort_unstable_by_key(|(idx, _)| *idx);
debug_assert_eq!(prepared.len(), batch_size);
histogram!("capture_kafka_batch_prep_duration_seconds")
.record(prep_start.elapsed().as_secs_f64());
Ok(prepared.into_iter().map(|(_, payload)| payload).collect())
}
}
#[async_trait]
impl<P: KafkaProducer + 'static> PublishPayloads for KafkaSinkBase<P> {
/// Serial enqueue in payload order + fail-fast ack drain. The serial
/// enqueue is the ordering bottleneck we deliberately keep: librdkafka
/// preserves per-partition on-wire order by send_result() call order, and
/// same-distinct_id events hash to the same partition via murmur2.
/// Within-batch same-key ordering must survive so e.g. $identify lands
/// before subsequent events.
///
/// Results are batch-uniform: the whole batch reports the first failure,
/// acked-before-failure events included. `publish_prepared` reports per event.
async fn publish(&self, payloads: Vec<PreparedPayload>) -> Vec<SinkResult> {
// Results are built in payload order as the loop enqueues, so the one
// Vec the return type requires is the only allocation on the happy
// path. Failure paths rewrite it in place to the batch-uniform error.
let mut results: Vec<SinkResult> = Vec::with_capacity(payloads.len());
let enqueue_start = Instant::now();
let mut ack_set = AckSet::new();
let mut payloads = payloads.into_iter();
for payload in payloads.by_ref() {
let uuid = payload.uuid;
match self.enqueue_record(payload) {
Ok(ack_future) => {
ack_set.spawn(ack_future);
results.push(SinkResult::published(uuid));
}
Err(err) => {
ack_set.cancel();
// Record enqueue duration on the error path too so slow-fail
// cases (e.g. QueueFull after a long stall) stay observable.
histogram!("capture_kafka_batch_enqueue_duration_seconds")
.record(enqueue_start.elapsed().as_secs_f64());
for result in &mut results {
result.outcome = Outcome::Failed(err.clone());
}
results.push(SinkResult::failed(uuid, err.clone()));
results.extend(payloads.map(|p| SinkResult::failed(p.uuid, err.clone())));
return results;
}
}
}
histogram!("capture_kafka_batch_enqueue_duration_seconds")
.record(enqueue_start.elapsed().as_secs_f64());
match ack_set.wait().await {
Ok(()) => results,
Err(err) => {
for result in &mut results {
result.outcome = Outcome::Failed(err.clone());
}
results
}
}
}
}
#[async_trait]
impl<P: KafkaProducer + 'static> PublishEvents for KafkaSinkBase<P> {
#[instrument(skip_all)]
async fn publish_events(&self, events: Vec<ProcessedEvent>) -> Result<(), CaptureError> {
// Skip `prepare_batch`, a `JoinSet` of one, and a fold over one
// result on the endpoints that publish a single event per request.
if events.len() == 1 {
let event = events.into_iter().next().expect("length checked above");
return self
.kafka_send(event)?
.instrument(info_span!("ack_wait_one"))
.await;
}
let payloads = self.prepare_batch(events).await?;
fold_results(PublishPayloads::publish(self, payloads).await)
}
}
#[async_trait]
impl<P: KafkaProducer + 'static> PublishPrepared for KafkaSinkBase<P> {
#[instrument(skip_all)]
async fn publish_prepared(&self, events: Vec<PreparedEvent>) -> Vec<SinkResult> {
let enqueue_start = Instant::now();
let enqueued: Vec<_> = events
.into_iter()
.map(|event| {
let uuid = event.uuid;
let ack = self.send(
&Destination::for_address(event.address),
event.partition_key,
event.ordering,
event.payload,
event.headers,
);
(uuid, ack)
})
.collect();
histogram!("capture_kafka_batch_enqueue_duration_seconds")
.record(enqueue_start.elapsed().as_secs_f64());
join_all(enqueued.into_iter().map(|(uuid, ack)| async move {
match ack {
Ok(ack) => match ack.await {
Ok(()) => SinkResult::published(uuid),
Err(err) => SinkResult::failed(uuid, err),
},
Err(err) => SinkResult::failed(uuid, err),
}
}))
.instrument(info_span!("ack_wait_many"))
.await
}
}
/// The ack futures of one `PublishPayloads::publish` batch, awaited concurrently.
struct AckSet(JoinSet<Result<(), CaptureError>>);
impl AckSet {
fn new() -> Self {
Self(JoinSet::new())
}
fn spawn<F>(&mut self, ack: F)
where
F: Future<Output = Result<(), CaptureError>> + Send + 'static,
{
self.0.spawn(ack);
}
/// Waits for every ack. The first failure cancels the rest.
async fn wait(mut self) -> Result<(), CaptureError> {
async move {
while let Some(res) = self.0.join_next().await {
match res {
Ok(Ok(_)) => {}
Ok(Err(err)) => {
self.cancel();
return Err(err);
}
Err(err) => {
self.cancel();
error!("join error while waiting on Kafka ACK: {err:#}");
return Err(CaptureError::RetryableSinkError);
}
}
}
Ok(())
}
.instrument(info_span!("ack_wait_many"))
.await
}
/// Cancels every unfinished ack. A cancelled `DeliveryAckFuture` records
/// its wait as outcome `dropped`.
fn cancel(mut self) {
self.0.abort_all();
}
}
#[cfg(test)]
pub(crate) use crate::sinks::registry::test_outputs;
#[cfg(test)]
mod tests {
use crate::api::CaptureError;
use crate::config::EnvelopeCompression;
use crate::outputs::PublishEvents;
use crate::producers::{self, ProducerName, ProducerRegistry};
use crate::sinks::kafka::{test_outputs, KafkaSink};
use crate::utils::uuid_v7_from_datetime;
use crate::v0_request::{DataType, OverflowReason, ProcessedEvent, ProcessedEventMetadata};
use common_types::CapturedEvent;
use rand::distributions::Alphanumeric;
use rand::Rng;
use rdkafka::mocking::MockCluster;
use rdkafka::producer::DefaultProducerContext;
use rdkafka::types::{RDKafkaApiKey, RDKafkaRespErr};
use std::collections::HashMap;
use tokio_util::sync::CancellationToken;
async fn start_on_mocked_sink(
message_max_bytes: Option<u32>,
) -> (MockCluster<'static, DefaultProducerContext>, KafkaSink) {
let shutdown_token = CancellationToken::new();
let mut manager = lifecycle::Manager::builder("test")
.with_trap_signals(false)
.with_prestop_check(false)
.with_shutdown_token(shutdown_token)
.build();
let handle = manager.register(
"sink",
lifecycle::ComponentOptions::new()
.with_liveness_deadline(std::time::Duration::from_secs(30)),
);
let _monitor = manager.monitor_background();
let cluster = MockCluster::new(1).expect("failed to create mock brokers");
let message_max_bytes = message_max_bytes.unwrap_or(1000000).to_string();
let env: HashMap<String, String> = [
("METADATA_BROKER_LIST", cluster.bootstrap_servers()),
("LINGER_MS", "0".to_string()),
("QUEUE_BUFFERING_MAX_KBYTES", (50 * 1024).to_string()),
("MESSAGE_TIMEOUT_MS", "500".to_string()),
("MESSAGE_MAX_BYTES", message_max_bytes),
]
.into_iter()
.map(|(k, v)| (format!("KAFKA_INGESTION_PRODUCER_{k}"), v))
.collect();
let producers = ProducerRegistry::build(
&producers::load_all(&env).expect("invalid producer config"),
HashMap::from([(ProducerName::Ingestion, handle)]),
)
.expect("failed to create producer");
let sink = KafkaSink::new(
test_outputs().map_producers(|name| producers.get(*name)),
EnvelopeCompression::None,
);
(cluster, sink)
}
#[tokio::test]
async fn kafka_sink_error_handling() {
// Uses a mocked Kafka broker that allows injecting write errors, to check error handling.
// We test different cases in a single test to amortize the startup cost of the producer.
let (cluster, sink) = start_on_mocked_sink(Some(3000000)).await;
let distinct_id = "test_distinct_id_123".to_string();
let timestamp = chrono::Utc::now();
let event: CapturedEvent = CapturedEvent {
uuid: uuid_v7_from_datetime(timestamp),
distinct_id: distinct_id.clone(),
session_id: None,
ip: "".to_string(),
data: "".to_string(),
now: "".to_string(),
sent_at: None,
token: "token1".to_string(),
event: "test_event".to_string(),
timestamp,
is_cookieless_mode: false,
historical_migration: false,
};
let metadata = ProcessedEventMetadata {
data_type: DataType::AnalyticsMain,
session_id: None,
computed_timestamp: None,
event_name: "test_event".to_string(),
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
skip_heatmap_processing: false,
overflow_reason: None,
distinct_id_truncated_from: None,
};
let event = ProcessedEvent {
event,
metadata: metadata.clone(),
};
// Wait for producer to be healthy, to keep kafka_message_timeout_ms short and tests faster
for _ in 0..20 {
if sink.publish_events(vec![event.clone()]).await.is_ok() {
break;
}
}
sink.publish_events(vec![event.clone()])
.await
.expect("failed to send one initial event");
sink.publish_events(vec![event.clone(), event.clone()])
.await
.expect("failed to send initial event batch");
// Producer should accept a 2MB message as we set message.max.bytes to 3MB
let big_data = rand::thread_rng()
.sample_iter(Alphanumeric)
.take(2_000_000)
.map(char::from)
.collect();
let timestamp = chrono::Utc::now();
let captured = CapturedEvent {
uuid: uuid_v7_from_datetime(timestamp),
distinct_id: "id1".to_string(),
session_id: None,
ip: "".to_string(),
data: big_data,
now: "".to_string(),
sent_at: None,
token: "token1".to_string(),
event: "test_event".to_string(),
timestamp,
is_cookieless_mode: false,
historical_migration: false,
};
let big_event = ProcessedEvent {
event: captured,
metadata: metadata.clone(),
};
sink.publish_events(vec![big_event])
.await
.expect("failed to send event larger than default max size");
// Producer should reject a 4MB message
let big_data = rand::thread_rng()
.sample_iter(Alphanumeric)
.take(4_000_000)
.map(char::from)
.collect();
let timestamp = chrono::Utc::now();
let big_event = ProcessedEvent {
event: CapturedEvent {
uuid: uuid_v7_from_datetime(timestamp),
distinct_id: "id1".to_string(),
session_id: None,
ip: "".to_string(),
data: big_data,
now: "".to_string(),
sent_at: None,
token: "token1".to_string(),
event: "test_event".to_string(),
timestamp,
is_cookieless_mode: false,
historical_migration: false,
},
metadata: metadata.clone(),
};
match sink.publish_events(vec![big_event]).await {
Err(CaptureError::EventTooBig(_)) => {} // Expected
Err(err) => panic!("wrong error code {err}"),
Ok(()) => panic!("should have errored"),
};
// Simulate unretriable errors
cluster.clear_request_errors(RDKafkaApiKey::Produce);
let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE; 1];
cluster.request_errors(RDKafkaApiKey::Produce, &err);
match sink.publish_events(vec![event.clone()]).await {
Err(CaptureError::EventTooBig(_)) => {} // Expected
Err(err) => panic!("wrong error code {err}"),
Ok(()) => panic!("should have errored"),
};
cluster.clear_request_errors(RDKafkaApiKey::Produce);
let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_INVALID_PARTITIONS; 1];
cluster.request_errors(RDKafkaApiKey::Produce, &err);
match sink
.publish_events(vec![event.clone(), event.clone()])
.await
{
Err(CaptureError::RetryableSinkError) => {} // Expected
Err(err) => panic!("wrong error code {err}"),
Ok(()) => panic!("should have errored"),
};
// Simulate transient errors, messages should go through OK
cluster.clear_request_errors(RDKafkaApiKey::Produce);
let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE; 2];
cluster.request_errors(RDKafkaApiKey::Produce, &err);
sink.publish_events(vec![event.clone()])
.await
.expect("failed to send one event after recovery");
cluster.clear_request_errors(RDKafkaApiKey::Produce);
let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE; 2];
cluster.request_errors(RDKafkaApiKey::Produce, &err);
sink.publish_events(vec![event.clone(), event.clone()])
.await
.expect("failed to send event batch after recovery");
// Timeout on a sustained transient error
cluster.clear_request_errors(RDKafkaApiKey::Produce);
let err = [RDKafkaRespErr::RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE; 50];
cluster.request_errors(RDKafkaApiKey::Produce, &err);
match sink.publish_events(vec![event.clone()]).await {
Err(CaptureError::RetryableSinkError) => {} // Expected
Err(err) => panic!("wrong error code {err}"),
Ok(()) => panic!("should have errored"),
};
match sink
.publish_events(vec![event.clone(), event.clone()])
.await
{
Err(CaptureError::RetryableSinkError) => {} // Expected
Err(err) => panic!("wrong error code {err}"),
Ok(()) => panic!("should have errored"),
};
}
#[tokio::test]
async fn test_historical_migration_headers() {
use common_types::CapturedEventHeaders;
use rdkafka::message::OwnedHeaders;
// Test that historical_migration=true is set in headers for AnalyticsHistorical
let headers_historical = CapturedEventHeaders {
token: Some("test_token".to_string()),
distinct_id: Some("test_id".to_string()),
session_id: None,
timestamp: Some("2023-01-01T12:00:00Z".to_string()),
event: Some("test_event".to_string()),
uuid: Some("test-uuid".to_string()),
now: Some("2023-01-01T12:00:00Z".to_string()),
force_disable_person_processing: None,
historical_migration: Some(true),
skip_heatmap_processing: None,
dlq_reason: None,
dlq_step: None,
dlq_timestamp: None,
content_encoding: None,
};
let owned_headers: OwnedHeaders = headers_historical.into();
let parsed_headers = CapturedEventHeaders::from(owned_headers);
assert_eq!(parsed_headers.historical_migration, Some(true));
assert_eq!(parsed_headers.now, Some("2023-01-01T12:00:00Z".to_string()));
let headers_main = CapturedEventHeaders {
token: Some("test_token".to_string()),
distinct_id: Some("test_id".to_string()),
session_id: None,
timestamp: Some("2023-01-01T12:00:00Z".to_string()),
event: Some("test_event".to_string()),
uuid: Some("test-uuid".to_string()),
now: Some("2023-01-01T12:00:00Z".to_string()),
force_disable_person_processing: None,
historical_migration: Some(false),
skip_heatmap_processing: None,
dlq_reason: None,
dlq_step: None,
dlq_timestamp: None,
content_encoding: None,
};
let owned_headers: OwnedHeaders = headers_main.into();
let parsed_headers = CapturedEventHeaders::from(owned_headers);
assert_eq!(parsed_headers.historical_migration, Some(false));
assert_eq!(parsed_headers.now, Some("2023-01-01T12:00:00Z".to_string()));
}
#[tokio::test]
async fn test_now_header_is_set() {
use common_types::CapturedEventHeaders;
use rdkafka::message::OwnedHeaders;
let test_now = "2024-01-15T10:30:45Z".to_string();
let headers = CapturedEventHeaders {
token: Some("test_token".to_string()),
distinct_id: Some("test_id".to_string()),
session_id: None,
timestamp: Some("2024-01-15T10:30:00Z".to_string()),
event: Some("test_event".to_string()),
uuid: Some("test-uuid".to_string()),
now: Some(test_now.clone()),
force_disable_person_processing: None,
historical_migration: None,
skip_heatmap_processing: None,
dlq_reason: None,
dlq_step: None,
dlq_timestamp: None,
content_encoding: None,
};
let owned_headers: OwnedHeaders = headers.into();
let parsed_headers = CapturedEventHeaders::from(owned_headers);
assert_eq!(parsed_headers.now, Some(test_now));
assert_eq!(parsed_headers.token, Some("test_token".to_string()));
assert_eq!(parsed_headers.distinct_id, Some("test_id".to_string()));
}
#[tokio::test]
async fn test_dlq_headers_are_set() {
use common_types::CapturedEventHeaders;
use rdkafka::message::OwnedHeaders;
let test_now = "2024-01-15T10:30:45Z".to_string();
let dlq_timestamp = "2025-01-15T10:30:45Z".to_string();
let headers = CapturedEventHeaders {
token: Some("test_token".to_string()),
distinct_id: Some("test_id".to_string()),
session_id: None,
timestamp: Some("2024-01-15T10:30:00Z".to_string()),
event: Some("test_event".to_string()),
uuid: Some("test-uuid".to_string()),
now: Some(test_now.clone()),
force_disable_person_processing: None,
historical_migration: None,
skip_heatmap_processing: None,
dlq_reason: Some("test reason".to_string()),
dlq_step: Some("test step".to_string()),
dlq_timestamp: Some(dlq_timestamp.clone()),
content_encoding: None,
};
let owned_headers: OwnedHeaders = headers.into();
let parsed_headers = CapturedEventHeaders::from(owned_headers);
assert_eq!(parsed_headers.dlq_reason, Some("test reason".to_string()));
assert_eq!(parsed_headers.dlq_step, Some("test step".to_string()));
assert_eq!(parsed_headers.dlq_timestamp, Some(dlq_timestamp));
}
#[cfg(test)]
mod topic_routing {
use super::*;
use crate::sinks::kafka::{test_outputs, KafkaSinkBase, SCATTER_GATHER_MIN_BATCH};
use crate::sinks::producer::MockKafkaProducer;
use crate::sinks::sink::{Outcome, PublishPayloads};
use rstest::rstest;
const MAIN_TOPIC: &str = "events_plugin_ingestion";
const OVERFLOW_TOPIC: &str = "events_plugin_ingestion_overflow";
const DLQ_TOPIC: &str = "events_plugin_ingestion_dlq";
const HISTORICAL_TOPIC: &str = "events_plugin_ingestion_historical";
const HEATMAPS_TOPIC: &str = "heatmaps";
const CLIENT_INGESTION_WARNING_TOPIC: &str = "client_ingestion_warning";
const REPLAY_OVERFLOW_TOPIC: &str = "replay_overflow";
const ERROR_TRACKING_TOPIC: &str = "error_tracking_events";
const AI_EVENTS_TOPIC: &str = "ai_events";
const AI_EVENTS_OVERFLOW_TOPIC: &str = "ai_events_overflow";
/// Which reroute counter (if any) an event must increment. DLQ and
/// custom-topic redirects are mutually exclusive at the sink because DLQ
/// takes strict priority, so a single event fires at most one counter.
#[derive(Clone, Copy, PartialEq, Debug)]
enum Rerouted {
None,
Dlq,
CustomTopic,
}
struct EventInput {
data_type: DataType,
force_overflow: bool,
skip_person_processing: bool,
skip_heatmap_processing: bool,
redirect_to_dlq: bool,
redirect_to_topic: Option<String>,
overflow_reason: Option<OverflowReason>,
compression: EnvelopeCompression,
}
impl Default for EventInput {
fn default() -> Self {
Self {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
skip_heatmap_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
compression: EnvelopeCompression::None,
}
}
}
fn create_test_event(input: &EventInput) -> ProcessedEvent {
let timestamp = chrono::Utc::now();
let event = CapturedEvent {
uuid: uuid_v7_from_datetime(timestamp),
distinct_id: "test_user".to_string(),
session_id: Some("session123".to_string()),
ip: "127.0.0.1".to_string(),
data: "{}".to_string(),
now: "2024-01-01T00:00:00Z".to_string(),
sent_at: None,
token: "test_token".to_string(),
event: "test_event".to_string(),
timestamp,
is_cookieless_mode: false,
historical_migration: false,
};
let metadata = ProcessedEventMetadata {
data_type: input.data_type,
session_id: Some("session123".to_string()),
computed_timestamp: None,
event_name: "test_event".to_string(),
force_overflow: input.force_overflow,
skip_person_processing: input.skip_person_processing,
redirect_to_dlq: input.redirect_to_dlq,
redirect_to_topic: input.redirect_to_topic.clone(),
skip_heatmap_processing: input.skip_heatmap_processing,
overflow_reason: input.overflow_reason.clone(),
distinct_id_truncated_from: None,
};
ProcessedEvent { event, metadata }
}
/// The full routing fingerprint of one event: topic + partition key +
/// every header the sink can stamp + which reroute counter (if any)
/// fires. This is the golden oracle produce-path refactors prove wire
/// parity against, so each field is stated explicitly rather than
/// re-derived from the input. Fields default to the "normal main-topic" outcome so
/// each case only spells out what makes it different.
struct ExpectedRouting<'a> {
topic: &'a str,
has_key: bool,
force_disable_person_processing: Option<bool>,
skip_heatmap_processing: Option<bool>,
content_encoding: Option<&'a str>,
dlq_headers: bool,
rerouted: Rerouted,
}
impl Default for ExpectedRouting<'_> {
fn default() -> Self {
Self {
topic: "",
has_key: true,
force_disable_person_processing: None,
skip_heatmap_processing: None,
content_encoding: None,
dlq_headers: false,
rerouted: Rerouted::None,
}
}
}
async fn assert_routing(input: EventInput, expected: ExpectedRouting<'_>) {
// Capture reroute counters on a thread-local recorder. `assert_routing`
// runs on the current-thread test runtime and `kafka_send` prepares
// the record inline before the first await, so the guard stays visible
// when `prepare_record` increments the counter.
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let _guard = metrics::set_default_local_recorder(&recorder);
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer_and_compression(
producer.clone(),
test_outputs(),
input.compression,
);
let event = create_test_event(&input);
sink.publish_events(vec![event]).await.unwrap();
let records = producer.get_records();
assert_eq!(records.len(), 1, "Expected exactly one record");
let record = &records[0];
let headers = &record.headers;
let ctx = format!(
"{:?} (force_overflow={}, skip_person={}, skip_heatmap={}, dlq={}, redirect_to_topic={:?}, overflow_reason={:?})",
input.data_type,
input.force_overflow,
input.skip_person_processing,
input.skip_heatmap_processing,
input.redirect_to_dlq,
input.redirect_to_topic,
input.overflow_reason,
);
assert_eq!(&*record.topic, expected.topic, "wrong topic for {ctx}");
assert_eq!(
record.key.is_some(),
expected.has_key,
"wrong key presence for {ctx}"
);
assert_eq!(
headers.force_disable_person_processing, expected.force_disable_person_processing,
"wrong force_disable_person_processing header for {ctx}"
);
assert_eq!(
headers.skip_heatmap_processing, expected.skip_heatmap_processing,
"wrong skip_heatmap_processing header for {ctx}"
);
assert_eq!(
headers.content_encoding.as_deref(),
expected.content_encoding,
"wrong content_encoding header for {ctx}"
);
// DLQ headers travel as a set: a reason, a step, and a valid RFC-3339
// timestamp when the event is routed to the DLQ; all three absent on
// every other route.
if expected.dlq_headers {
assert_eq!(
headers.dlq_reason.as_deref(),
Some("event_restriction"),
"wrong dlq_reason for {ctx}"
);
assert_eq!(
headers.dlq_step.as_deref(),
Some("capture"),
"wrong dlq_step for {ctx}"
);
let ts = headers
.dlq_timestamp
.as_deref()
.unwrap_or_else(|| panic!("dlq_timestamp missing for {ctx}"));
chrono::DateTime::parse_from_rfc3339(ts).unwrap_or_else(|e| {
panic!("dlq_timestamp '{ts}' is not valid RFC 3339 for {ctx}: {e}")
});
} else {
assert_eq!(
headers.dlq_reason, None,
"dlq_reason must be absent for {ctx}"
);
assert_eq!(headers.dlq_step, None, "dlq_step must be absent for {ctx}");
assert_eq!(
headers.dlq_timestamp, None,
"dlq_timestamp must be absent for {ctx}"
);
}
// Exactly one reroute counter fires per redirected event; neither
// fires on the normal per-datatype or overflow paths.
let snapshot = snapshotter.snapshot().into_vec();
let count = |name: &str| -> Option<u64> {
snapshot.iter().find_map(|(key, _, _, value)| {
if key.key().name() != name {
return None;
}
match value {
metrics_util::debugging::DebugValue::Counter(v) => Some(*v),
_ => None,
}
})
};
// Recorder liveness: enqueue emits the bytes counter for every
// record, on the same inline pre-await path as the reroute
// counters. If it is absent, the thread-local recorder is no
// longer observing prep (e.g. prep moved to a spawned thread)
// and every absence assertion below would pass vacuously — so
// fail here first, in every case.
assert_eq!(
count("capture_kafka_produce_bytes_total"),
Some(record.payload.len() as u64),
"recorder did not observe the produce path for {ctx}; \
the counter assertions below cannot be trusted"
);
let dlq_count = count("capture_events_rerouted_dlq");
let custom_count = count("capture_events_rerouted_custom_topic");
match expected.rerouted {
Rerouted::None => {
assert_eq!(
dlq_count, None,
"capture_events_rerouted_dlq must not fire for {ctx}"
);
assert_eq!(
custom_count, None,
"capture_events_rerouted_custom_topic must not fire for {ctx}"
);
}
Rerouted::Dlq => {
assert_eq!(
dlq_count,
Some(1),
"capture_events_rerouted_dlq must fire once for {ctx}"
);
assert_eq!(
custom_count, None,
"capture_events_rerouted_custom_topic must not fire for {ctx}"
);
}
Rerouted::CustomTopic => {
assert_eq!(
custom_count,
Some(1),
"capture_events_rerouted_custom_topic must fire once for {ctx}"
);
assert_eq!(
dlq_count, None,
"capture_events_rerouted_dlq must not fire for {ctx}"
);
}
}
}
// ==================== AnalyticsMain ====================
#[tokio::test]
async fn analytics_main_normal() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_force_overflow() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_force_overflow_with_skip_person() {
// Key should be dropped when both force_overflow and skip_person_processing are set
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: OVERFLOW_TOPIC,
has_key: false,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_skip_person_only() {
// Key should be dropped when skip_person_processing is set, even without overflow
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
has_key: false,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_dlq_priority_over_overflow() {
// DLQ takes priority over force_overflow
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_dlq_with_skip_person() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_all_flags() {
// DLQ takes priority, skip_person still sets header
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
skip_person_processing: true,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
// ==================== AnalyticsHistorical ====================
// Historical events IGNORE force_overflow - they never overflow
#[tokio::test]
async fn analytics_historical_normal() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsHistorical,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: HISTORICAL_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_historical_ignores_force_overflow() {
// Historical events should ignore force_overflow
assert_routing(
EventInput {
data_type: DataType::AnalyticsHistorical,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: HISTORICAL_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_historical_skip_person() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsHistorical,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: HISTORICAL_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_historical_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsHistorical,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_historical_all_flags() {
// DLQ takes priority, historical ignores overflow, skip_person sets header
assert_routing(
EventInput {
data_type: DataType::AnalyticsHistorical,
force_overflow: true,
skip_person_processing: true,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
// ==================== SnapshotMain ====================
#[tokio::test]
async fn snapshot_normal() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_force_overflow() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: REPLAY_OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_force_overflow_with_skip_person() {
// Unlike AnalyticsMain, SnapshotMain does NOT drop key with skip_person_processing
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: true,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: REPLAY_OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_skip_person_only() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_dlq_priority_over_overflow() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_lz4_sets_content_encoding_header() {
// With envelope compression on, a snapshot event carries the
// `content-encoding: lz4` header alongside its normal main-topic
// routing. The compressed-payload bytes are covered by the lz4
// payload goldens below; here the oracle pins just the header.
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
compression: EnvelopeCompression::Lz4,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
content_encoding: Some("lz4"),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_lz4_leaves_content_encoding_unset() {
// Envelope compression only applies to snapshots: a non-snapshot
// event under the same sink config carries no content-encoding.
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
compression: EnvelopeCompression::Lz4,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
..Default::default()
},
)
.await;
}
// ==================== HeatmapMain ====================
// Heatmaps IGNORE force_overflow
#[tokio::test]
async fn heatmap_normal() {
assert_routing(
EventInput {
data_type: DataType::HeatmapMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: HEATMAPS_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn heatmap_ignores_force_overflow() {
assert_routing(
EventInput {
data_type: DataType::HeatmapMain,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: HEATMAPS_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn heatmap_skip_person() {
assert_routing(
EventInput {
data_type: DataType::HeatmapMain,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: HEATMAPS_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn heatmap_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::HeatmapMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn heatmap_skip_heatmap_processing_sets_header() {
// The skip_heatmap_processing metadata flag stamps its own header,
// independent of the routing topic and the person-processing flag.
assert_routing(
EventInput {
data_type: DataType::HeatmapMain,
skip_heatmap_processing: true,
..Default::default()
},
ExpectedRouting {
topic: HEATMAPS_TOPIC,
skip_heatmap_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_skip_heatmap_processing_sets_header() {
// skip_heatmap_processing is not gated on data type: it rides through
// for AnalyticsMain too, orthogonally to skip_person_processing.
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
skip_heatmap_processing: true,
skip_person_processing: true,
..Default::default()
},
ExpectedRouting {
topic: MAIN_TOPIC,
has_key: false,
force_disable_person_processing: Some(true),
skip_heatmap_processing: Some(true),
..Default::default()
},
)
.await;
}
// ==================== ExceptionErrorTracking ====================
// Exceptions IGNORE force_overflow
#[tokio::test]
async fn exception_normal() {
assert_routing(
EventInput {
data_type: DataType::ExceptionErrorTracking,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: ERROR_TRACKING_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn exception_ignores_force_overflow() {
assert_routing(
EventInput {
data_type: DataType::ExceptionErrorTracking,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: ERROR_TRACKING_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn exception_skip_person() {
assert_routing(
EventInput {
data_type: DataType::ExceptionErrorTracking,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: ERROR_TRACKING_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn exception_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::ExceptionErrorTracking,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
// ==================== ClientIngestionWarning ====================
// ClientIngestionWarning IGNORES force_overflow
#[tokio::test]
async fn client_ingestion_warning_normal() {
assert_routing(
EventInput {
data_type: DataType::ClientIngestionWarning,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: CLIENT_INGESTION_WARNING_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn client_ingestion_warning_ignores_force_overflow() {
assert_routing(
EventInput {
data_type: DataType::ClientIngestionWarning,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: CLIENT_INGESTION_WARNING_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn client_ingestion_warning_skip_person() {
assert_routing(
EventInput {
data_type: DataType::ClientIngestionWarning,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: CLIENT_INGESTION_WARNING_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn client_ingestion_warning_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::ClientIngestionWarning,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
// ==================== AiEvents ====================
#[tokio::test]
async fn ai_events_normal() {
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: AI_EVENTS_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn ai_events_force_overflow_reroutes() {
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: AI_EVENTS_OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn ai_events_force_overflow_with_skip_person_drops_key() {
// Mirrors analytics_main_force_overflow_with_skip_person.
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: true,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: AI_EVENTS_OVERFLOW_TOPIC,
has_key: false,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
/// Stamped overflow reasons on the AI lane, where — unlike the
/// analytics lane — a burst without locality preservation spreads
/// while person processing is on: the AI consumer reads persons
/// without writing them, so keyless person-on records contend
/// nothing downstream. `ForceLimited` implies the person-processing
/// header on its own, flag or no flag.
#[rstest]
#[case::force_limited(OverflowReason::ForceLimited, false, Some(true))]
#[case::rate_limited_preserving(
OverflowReason::RateLimited {
preserve_locality: true
},
true,
None
)]
#[case::rate_limited_spreading(
OverflowReason::RateLimited {
preserve_locality: false
},
false,
None
)]
#[tokio::test]
async fn ai_events_stamped_overflow_routing(
#[case] reason: OverflowReason,
#[case] has_key: bool,
#[case] force_disable_person_processing: Option<bool>,
) {
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: Some(reason),
..Default::default()
},
ExpectedRouting {
topic: AI_EVENTS_OVERFLOW_TOPIC,
has_key,
force_disable_person_processing,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn ai_events_without_overflow_lane_never_overflow() {
// Import has no AI overflow lane, so every overflow stamp is ignored.
let producer = MockKafkaProducer::new();
let mut topics = test_outputs();
topics.has_ai_overflow_lane = false;
let sink = KafkaSinkBase::with_producer(producer.clone(), topics);
let mut event = create_test_event(&EventInput {
data_type: DataType::AiEvents,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
});
event.metadata.overflow_reason = Some(OverflowReason::ForceLimited);
sink.publish_events(vec![event]).await.unwrap();
let records = producer.get_records();
assert_eq!(records.len(), 1);
assert_eq!(&*records[0].topic, AI_EVENTS_TOPIC);
assert_eq!(records[0].key.as_deref(), Some("test_token:test_user"));
}
#[tokio::test]
async fn ai_events_skip_person_keeps_key() {
// skip_person_processing sets the header but must not null the
// key: the AI main lane keeps per-distinct-id ordering.
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: AI_EVENTS_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn ai_events_redirect_to_dlq() {
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn ai_events_redirect_to_topic_wins() {
// A restriction-driven redirect beats the AI lane, matching v1
// where Destination::Custom overwrites Destination::AiMain.
assert_routing(
EventInput {
data_type: DataType::AiEvents,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: "custom_topic",
has_key: true,
force_disable_person_processing: None,
rerouted: Rerouted::CustomTopic,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn ai_events_record_matches_other_lanes_byte_for_byte() {
// The AI lane must only change the topic: for the same event, the
// record key is the event key (token:distinct_id) and the headers
// are identical to what another dedicated lane produces.
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let base = create_test_event(&EventInput::default());
let mut ai_event = base.clone();
ai_event.metadata.data_type = DataType::AiEvents;
let mut exception_event = base;
exception_event.metadata.data_type = DataType::ExceptionErrorTracking;
sink.publish_events(vec![ai_event]).await.unwrap();
sink.publish_events(vec![exception_event]).await.unwrap();
let records = producer.get_records();
assert_eq!(records.len(), 2);
assert_eq!(&*records[0].topic, AI_EVENTS_TOPIC);
assert_eq!(&*records[1].topic, ERROR_TRACKING_TOPIC);
assert_eq!(records[0].key.as_deref(), Some("test_token:test_user"));
assert_eq!(records[0].key, records[1].key);
assert_eq!(records[0].payload, records[1].payload);
assert_eq!(
format!("{:?}", records[0].headers),
format!("{:?}", records[1].headers)
);
}
// ==================== RedirectToTopic ====================
// redirect_to_topic overrides normal routing but DLQ takes priority
#[tokio::test]
async fn analytics_main_redirect_to_topic() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: "custom_topic",
has_key: true,
force_disable_person_processing: None,
rerouted: Rerouted::CustomTopic,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_dlq_priority_over_redirect_to_topic() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: true,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: None,
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_redirect_to_topic_priority_over_overflow() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: "custom_topic",
has_key: true,
force_disable_person_processing: None,
rerouted: Rerouted::CustomTopic,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn analytics_main_redirect_to_topic_with_skip_person() {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: true,
redirect_to_dlq: false,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: "custom_topic",
has_key: true,
force_disable_person_processing: Some(true),
rerouted: Rerouted::CustomTopic,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn snapshot_redirect_to_topic() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: None,
..Default::default()
},
ExpectedRouting {
topic: "custom_topic",
has_key: true,
force_disable_person_processing: None,
rerouted: Rerouted::CustomTopic,
..Default::default()
},
)
.await;
}
// ==================== overflow_reason routing tests ====================
/// `ForceLimited` implies person processing is off on its own: the
/// header is set whether or not the stamping site also set the flag,
/// so a keyless force-limited record can never reach person
/// processing with identity resolution still on.
#[rstest]
#[case::stamped_with_flag(true)]
#[case::reason_only(false)]
#[tokio::test]
async fn overflow_reason_force_limited_routes_to_overflow_with_null_key(
#[case] skip_person_processing: bool,
) {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
skip_person_processing,
overflow_reason: Some(OverflowReason::ForceLimited),
..Default::default()
},
ExpectedRouting {
topic: OVERFLOW_TOPIC,
has_key: false,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
/// A person-on burst keeps its key on the analytics lane regardless of
/// the locality preference: the overflow consumer updates persons
/// keyed on distinct id, and spreading one distinct id across
/// partitions contends those updates.
#[rstest]
#[case::preserving_locality(true)]
#[case::spreading(false)]
#[tokio::test]
async fn overflow_reason_rate_limited_keeps_key_while_person_processing_on(
#[case] preserve_locality: bool,
) {
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
overflow_reason: Some(OverflowReason::RateLimited { preserve_locality }),
..Default::default()
},
ExpectedRouting {
topic: OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
/// The wire outcome for the combination the global rate limiter and the
/// overflow limiter produce together (the GRL stamps the person flag,
/// the burst limiter overwrites the reason): the record keeps the
/// person-processing header and loses the partition key, on either
/// locality setting.
#[rstest]
#[case::analytics_preserving(DataType::AnalyticsMain, true, OVERFLOW_TOPIC)]
#[case::analytics_spreading(DataType::AnalyticsMain, false, OVERFLOW_TOPIC)]
#[case::ai_preserving(DataType::AiEvents, true, AI_EVENTS_OVERFLOW_TOPIC)]
#[tokio::test]
async fn overflow_reason_rate_limited_drops_key_when_person_off(
#[case] data_type: DataType,
#[case] preserve_locality: bool,
#[case] expected_topic: &str,
) {
assert_routing(
EventInput {
data_type,
skip_person_processing: true,
overflow_reason: Some(OverflowReason::RateLimited { preserve_locality }),
..Default::default()
},
ExpectedRouting {
topic: expected_topic,
has_key: false,
force_disable_person_processing: Some(true),
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn overflow_reason_ignored_for_analytics_historical() {
// historical events never go through overflow routing even if the
// upstream pipeline accidentally stamps one — be defensive.
assert_routing(
EventInput {
data_type: DataType::AnalyticsHistorical,
overflow_reason: Some(OverflowReason::RateLimited {
preserve_locality: false,
}),
..Default::default()
},
ExpectedRouting {
topic: HISTORICAL_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn overflow_reason_replay_limited_routes_snapshot_to_replay_overflow() {
assert_routing(
EventInput {
data_type: DataType::SnapshotMain,
overflow_reason: Some(OverflowReason::ReplayLimited),
..Default::default()
},
ExpectedRouting {
topic: REPLAY_OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn overflow_reason_force_overflow_short_circuits_overflow_reason() {
// force_overflow from an event restriction wins over a stamped
// overflow_reason.
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
overflow_reason: Some(OverflowReason::RateLimited {
preserve_locality: false,
}),
..Default::default()
},
ExpectedRouting {
topic: OVERFLOW_TOPIC,
has_key: true,
force_disable_person_processing: None,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn overflow_reason_redirect_to_dlq_wins_over_overflow_reason() {
// DLQ routing is the highest-priority routing decision: it wins
// over both force_overflow and overflow_reason. The
// person-processing header still travels with the ForceLimited
// reason — routing precedence changes the topic, not the skip
// (production stamps the flag alongside the reason anyway).
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
redirect_to_dlq: true,
overflow_reason: Some(OverflowReason::ForceLimited),
..Default::default()
},
ExpectedRouting {
topic: DLQ_TOPIC,
has_key: true,
force_disable_person_processing: Some(true),
dlq_headers: true,
rerouted: Rerouted::Dlq,
..Default::default()
},
)
.await;
}
#[tokio::test]
async fn overflow_reason_redirect_to_topic_wins_over_overflow_reason() {
// Custom topic redirect (set by event restrictions) also wins over
// overflow_reason since overflow decisions cannot compose with a
// hard-coded topic override.
assert_routing(
EventInput {
data_type: DataType::AnalyticsMain,
redirect_to_topic: Some("custom_topic".to_string()),
overflow_reason: Some(OverflowReason::ForceLimited),
..Default::default()
},
ExpectedRouting {
topic: "custom_topic",
has_key: true,
// The header travels with the reason regardless of the
// routing precedence, as in the dlq case above.
force_disable_person_processing: Some(true),
rerouted: Rerouted::CustomTopic,
..Default::default()
},
)
.await;
}
// ==================== publish_events ordering + error tests ====================
// These exercise the three-phase batch path: parallel prepare_record,
// serial enqueue_record, concurrent ack drain. The ordering test runs on
// a multi-thread runtime so phase 1 actually parallelizes across workers
// and we can detect if phase 2 is accidentally reordering records.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn publish_events_preserves_order_same_key() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
// 20 events, all sharing the same distinct_id (so they hash to the
// same partition via murmur2), each with a unique UUID so we can
// track input->output order through the pipeline.
let events: Vec<ProcessedEvent> = (0..20)
.map(|_| {
create_test_event(&EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
})
})
.collect();
let input_uuids: Vec<String> =
events.iter().map(|e| e.event.uuid.to_string()).collect();
sink.publish_events(events)
.await
.expect("publish_events failed");
let records = producer.get_records();
assert_eq!(records.len(), 20, "expected 20 records");
// Parse the UUID out of each record's serialized payload and compare
// against the input order. If phase 2 ever reorders enqueue calls,
// librdkafka's partition-order guarantee would be broken for same-key
// events and this assertion trips.
let output_uuids: Vec<String> = records
.iter()
.map(|r| {
let v: serde_json::Value =
serde_json::from_slice(&r.payload).expect("payload is valid json");
v.get("uuid")
.and_then(|u| u.as_str())
.expect("uuid field present")
.to_string()
})
.collect();
assert_eq!(
output_uuids, input_uuids,
"publish_events must preserve input order for same-key events"
);
// Sanity: all records share the same partition key.
let first_key = records[0].key.as_deref().expect("partition key set");
for r in &records {
assert_eq!(
r.key.as_deref(),
Some(first_key),
"all events should share partition key"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn publish_events_prep_error_aborts_batch() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
// Build a batch where event #3 is a SnapshotMain with session_id=None,
// which causes prepare_record to return MissingSessionId. The other
// events are valid AnalyticsMain. Since phase 2 only runs after all
// prep tasks complete, a prep error must short-circuit before any
// producer.send() call — so the mock producer should see zero records.
let mut events: Vec<ProcessedEvent> = (0..5)
.map(|_| {
create_test_event(&EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
})
})
.collect();
// Overwrite element [2] with a SnapshotMain event whose session_id
// metadata is None — prepare_record returns MissingSessionId at the
// session_id lookup in the SnapshotMain branch.
let mut bad = create_test_event(&EventInput {
data_type: DataType::SnapshotMain,
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
overflow_reason: None,
..Default::default()
});
bad.metadata.session_id = None;
events[2] = bad;
let res = sink.publish_events(events).await;
match res {
Err(CaptureError::MissingSessionId) => {}
Err(other) => panic!("expected MissingSessionId, got {other:?}"),
Ok(()) => panic!("expected publish_events to fail on prep error"),
}
let records = producer.get_records();
assert!(
records.is_empty(),
"no records should reach the producer when prep phase fails; got {} records",
records.len()
);
}
// ==================== publish_events fast-path + mid-batch failure tests ====================
/// Builds N AnalyticsMain events with sequential distinct_ids so each
/// record is individually identifiable in the mock producer's output.
fn build_batch(n: usize) -> Vec<ProcessedEvent> {
(0..n)
.map(|i| {
let mut e = create_test_event(&EventInput::default());
e.event.distinct_id = format!("user_{i}");
e
})
.collect()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn publish_events_mid_enqueue_failure_preserves_earlier_records() {
// Fail at phase-2 send #3 (0-indexed): events [0, 1, 2] should land
// in the mock, publish_events must return Err, and no event at index
// >= 3 should ever hit the producer. Batch size is well above the
// scatter-gather threshold so phase 2 runs post-parallel-prep.
const BATCH: usize = 10;
const FAIL_IDX: usize = 3;
let producer = MockKafkaProducer::new_failing_at(FAIL_IDX);
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let events = build_batch(BATCH);
let input_distinct_ids: Vec<String> =
events.iter().map(|e| e.event.distinct_id.clone()).collect();
let res = sink.publish_events(events).await;
match res {
Err(CaptureError::RetryableSinkError) => {}
Err(other) => panic!("expected RetryableSinkError, got {other:?}"),
Ok(()) => panic!("expected publish_events to fail on enqueue #{FAIL_IDX}"),
}
let records = producer.get_records();
assert_eq!(
records.len(),
FAIL_IDX,
"expected exactly {FAIL_IDX} records to reach producer before failure"
);
// Output distinct_ids should match input[..FAIL_IDX] in order:
// phase-2 is serial in input order, so the earlier records must
// be the first FAIL_IDX events of the input batch.
let output_distinct_ids: Vec<String> = records
.iter()
.map(|r| {
let v: serde_json::Value =
serde_json::from_slice(&r.payload).expect("payload is valid json");
v.get("distinct_id")
.and_then(|u| u.as_str())
.expect("distinct_id field present")
.to_string()
})
.collect();
assert_eq!(
output_distinct_ids,
input_distinct_ids[..FAIL_IDX],
"earlier records must preserve input order on mid-batch failure"
);
}
#[tokio::test]
async fn publish_events_single_event_skips_the_batch_path() {
// One event short-circuits to kafka_send: no prepare_batch, no
// JoinSet. This is the live path for the single-event endpoints.
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let events = build_batch(1);
sink.publish_events(events)
.await
.expect("publish_events failed");
let records = producer.get_records();
assert_eq!(records.len(), 1, "expected exactly one record");
assert_eq!(&*records[0].topic, MAIN_TOPIC);
}
#[tokio::test]
async fn publish_events_two_events_take_the_batch_path() {
// Two events is the smallest batch that still reaches
// prepare_batch. A one-event guard that grew an off-by-one would
// swallow the second event here and nowhere else.
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
sink.publish_events(build_batch(2))
.await
.expect("publish_events failed");
let records = producer.get_records();
assert_eq!(records.len(), 2, "both events must reach the producer");
}
/// Runs `f` under a local metrics recorder and counts the samples that
/// landed on `name`. The recorder is thread-scoped, so `f` has to drive
/// its own futures on this thread.
fn histogram_sample_count(name: &str, f: impl FnOnce()) -> usize {
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let _guard = metrics::set_default_local_recorder(&recorder);
f();
snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter_map(|(key, _, _, value)| match value {
DebugValue::Histogram(samples) if key.key().name() == name => {
Some(samples.len())
}
_ => None,
})
.sum()
}
#[test]
fn publish_events_one_event_feeds_the_phase_histograms() {
// The one-event path skips prepare_batch and PublishPayloads::publish, which
// own these two histograms, so it records them itself. Otherwise single-event
// endpoints drop out of the distribution and the quantiles shift.
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
for name in [
"capture_kafka_batch_prep_duration_seconds",
"capture_kafka_batch_enqueue_duration_seconds",
] {
let samples = histogram_sample_count(name, || {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer, test_outputs());
runtime
.block_on(sink.publish_events(build_batch(1)))
.expect("publish_events failed");
});
assert_eq!(samples, 1, "{name} must see the one-event publish");
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn publish_events_just_below_threshold_uses_serial_path() {
// batch_size = SCATTER_GATHER_MIN_BATCH - 1 takes the serial fast
// path. We can't observe "which path ran" directly, so we assert
// behavioral equivalence: N records, correct topic, input order.
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let size = SCATTER_GATHER_MIN_BATCH - 1;
let events = build_batch(size);
let input_distinct_ids: Vec<String> =
events.iter().map(|e| e.event.distinct_id.clone()).collect();
sink.publish_events(events)
.await
.expect("publish_events failed");
let records = producer.get_records();
assert_eq!(records.len(), size);
let output: Vec<String> = records
.iter()
.map(|r| {
let v: serde_json::Value =
serde_json::from_slice(&r.payload).expect("payload is valid json");
v["distinct_id"].as_str().unwrap().to_string()
})
.collect();
assert_eq!(
output, input_distinct_ids,
"serial path must preserve order"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn publish_events_at_threshold_uses_scatter_gather_path() {
// batch_size = SCATTER_GATHER_MIN_BATCH takes the scatter-gather
// path. Behavioral equivalence with the serial path must hold:
// same N records, same order, same topics.
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let size = SCATTER_GATHER_MIN_BATCH;
let events = build_batch(size);
let input_distinct_ids: Vec<String> =
events.iter().map(|e| e.event.distinct_id.clone()).collect();
sink.publish_events(events)
.await
.expect("publish_events failed");
let records = producer.get_records();
assert_eq!(records.len(), size);
let output: Vec<String> = records
.iter()
.map(|r| {
let v: serde_json::Value =
serde_json::from_slice(&r.payload).expect("payload is valid json");
v["distinct_id"].as_str().unwrap().to_string()
})
.collect();
assert_eq!(
output, input_distinct_ids,
"scatter-gather path must preserve input order after sort_unstable_by_key"
);
}
// ==================== Sink mechanism seam ====================
// The per-event result surface `PublishPayloads::publish` reports: uuid-aligned
// with the input payloads, batch-uniform on failure. `fold_results`
// discards this shape, so the publish_events tests above cannot see it.
#[tokio::test]
async fn publish_reports_uuid_aligned_results() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let payloads = sink
.prepare_batch(build_batch(3))
.await
.expect("prepare_batch failed");
let input_uuids: Vec<_> = payloads.iter().map(|p| p.uuid).collect();
let results = sink.publish(payloads).await;
assert_eq!(results.len(), 3);
for (result, uuid) in results.iter().zip(&input_uuids) {
assert_eq!(result.uuid, *uuid, "results must align with input order");
assert!(matches!(result.outcome, Outcome::Published));
}
}
#[tokio::test]
async fn publish_enqueue_failure_is_batch_uniform() {
const FAIL_IDX: usize = 1;
let producer = MockKafkaProducer::new_failing_at(FAIL_IDX);
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let payloads = sink
.prepare_batch(build_batch(3))
.await
.expect("prepare_batch failed");
let input_uuids: Vec<_> = payloads.iter().map(|p| p.uuid).collect();
let results = sink.publish(payloads).await;
// Every event reports the failure, the one enqueued before it
// included; only the records before the failing index reached
// the producer.
assert_eq!(results.len(), 3);
for (result, uuid) in results.iter().zip(&input_uuids) {
assert_eq!(result.uuid, *uuid, "results must align with input order");
assert!(matches!(
result.outcome,
Outcome::Failed(CaptureError::RetryableSinkError)
));
}
assert_eq!(producer.get_records().len(), FAIL_IDX);
}
#[tokio::test]
async fn publish_ack_failure_is_batch_uniform() {
const FAIL_IDX: usize = 1;
let producer = MockKafkaProducer::new_failing_ack_at(FAIL_IDX);
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let payloads = sink
.prepare_batch(build_batch(3))
.await
.expect("prepare_batch failed");
let input_uuids: Vec<_> = payloads.iter().map(|p| p.uuid).collect();
let results = sink.publish(payloads).await;
// The producer accepted every record — all three were enqueued —
// but one delivery report failed, so the whole batch reports the
// failure, acked-before-failure events included.
assert_eq!(results.len(), 3);
for (result, uuid) in results.iter().zip(&input_uuids) {
assert_eq!(result.uuid, *uuid, "results must align with input order");
assert!(matches!(
result.outcome,
Outcome::Failed(CaptureError::RetryableSinkError)
));
}
assert_eq!(producer.get_records().len(), 3);
}
/// Per-event-type topic routing is covered by `assert_routing` for
/// the single-event path. This test verifies routing survives the
/// batch path for a mixed batch of data types plus one force_overflow
/// AnalyticsMain — exercised on both the serial fast path (5 events)
/// and the scatter-gather path (10 events).
async fn mixed_datatypes_routing_for_batch(pad_to: usize) {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
// Core 5-event diverse batch.
let mut events: Vec<ProcessedEvent> = vec![
create_test_event(&EventInput {
data_type: DataType::AnalyticsMain,
..EventInput::default()
}),
create_test_event(&EventInput {
data_type: DataType::HeatmapMain,
..EventInput::default()
}),
create_test_event(&EventInput {
data_type: DataType::ExceptionErrorTracking,
..EventInput::default()
}),
create_test_event(&EventInput {
data_type: DataType::ClientIngestionWarning,
..EventInput::default()
}),
create_test_event(&EventInput {
data_type: DataType::AnalyticsMain,
force_overflow: true,
..EventInput::default()
}),
];
// Pad with AnalyticsMain events if caller wants to push the batch
// over SCATTER_GATHER_MIN_BATCH. Padding goes at the end so the
// first 5 per-event assertions line up regardless of batch size.
while events.len() < pad_to {
events.push(create_test_event(&EventInput::default()));
}
sink.publish_events(events)
.await
.expect("publish_events failed");
let records = producer.get_records();
assert_eq!(records.len(), pad_to.max(5));
// Per-index topic assertions (order-preserving: phase-2 is serial
// in input order on both paths).
assert_eq!(&*records[0].topic, MAIN_TOPIC, "event[0]: AnalyticsMain");
assert_eq!(&*records[1].topic, HEATMAPS_TOPIC, "event[1]: HeatmapMain");
assert_eq!(
&*records[2].topic, ERROR_TRACKING_TOPIC,
"event[2]: ExceptionErrorTracking"
);
assert_eq!(
&*records[3].topic, CLIENT_INGESTION_WARNING_TOPIC,
"event[3]: ClientIngestionWarning"
);
assert_eq!(
&*records[4].topic, OVERFLOW_TOPIC,
"event[4]: AnalyticsMain + force_overflow"
);
}
#[tokio::test]
async fn publish_events_mixed_datatypes_serial_path() {
// 5 events < SCATTER_GATHER_MIN_BATCH => serial fast path.
mixed_datatypes_routing_for_batch(5).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn publish_events_mixed_datatypes_scatter_gather_path() {
// 10 events >= SCATTER_GATHER_MIN_BATCH => scatter-gather path.
mixed_datatypes_routing_for_batch(10).await;
}
// ==================== Envelope compression ====================
#[tokio::test]
async fn snapshot_payload_uncompressed_by_default() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let event = create_test_event(&EventInput {
data_type: DataType::SnapshotMain,
..Default::default()
});
sink.kafka_send(event).unwrap().await.unwrap();
let records = producer.get_records();
assert_eq!(records.len(), 1);
// Payload must be valid UTF-8 JSON when compression is off.
let v: serde_json::Value = serde_json::from_slice(&records[0].payload)
.expect("uncompressed payload is valid json");
assert!(v.get("distinct_id").is_some());
}
#[tokio::test]
async fn snapshot_payload_lz4_compressed_when_enabled() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer_and_compression(
producer.clone(),
test_outputs(),
EnvelopeCompression::Lz4,
);
let event = create_test_event(&EventInput {
data_type: DataType::SnapshotMain,
..Default::default()
});
sink.kafka_send(event).unwrap().await.unwrap();
let records = producer.get_records();
assert_eq!(records.len(), 1);
// `content-encoding: lz4` header must be present.
assert_eq!(
records[0].headers.content_encoding.as_deref(),
Some("lz4"),
"expected content-encoding: lz4 header"
);
// Payload = 4-byte LE uncompressed size + LZ4 block data. Decompress and verify.
let payload = &records[0].payload;
let uncompressed_len = u32::from_le_bytes(payload[..4].try_into().unwrap()) as usize;
let decompressed = lz4::block::decompress(&payload[4..], Some(uncompressed_len as i32))
.expect("failed to decompress");
let v: serde_json::Value =
serde_json::from_slice(&decompressed).expect("decompressed payload is valid json");
assert!(v.get("distinct_id").is_some());
}
#[tokio::test]
async fn non_snapshot_payload_not_compressed_when_lz4_enabled() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer_and_compression(
producer.clone(),
test_outputs(),
EnvelopeCompression::Lz4,
);
let event = create_test_event(&EventInput {
data_type: DataType::AnalyticsMain,
..Default::default()
});
sink.kafka_send(event).unwrap().await.unwrap();
let records = producer.get_records();
assert_eq!(records.len(), 1);
// Non-snapshot payloads must stay uncompressed regardless of the flag.
let v: serde_json::Value =
serde_json::from_slice(&records[0].payload).expect("payload must be plain json");
assert!(v.get("distinct_id").is_some());
}
}
mod prepared_route {
use super::*;
use crate::ordering::OrderingGuarantee;
use crate::outputs::{PreparedEvent, PublishPrepared};
use crate::pipeline::{
Address, AiLane, AnalyticsLane, BasicLane, PipelineLane, SessionReplayLane,
};
use crate::sinks::kafka::KafkaSinkBase;
use crate::sinks::producer::MockKafkaProducer;
use crate::sinks::sink::{Outcome, SinkResult};
use common_types::CapturedEventHeaders;
use rstest::rstest;
use std::sync::Arc;
use uuid::Uuid;
fn prepared(address: Address, ordering: OrderingGuarantee) -> PreparedEvent {
let uuid = Uuid::now_v7();
PreparedEvent {
uuid,
address,
partition_key: "token:distinct_id".to_string(),
ordering,
payload: bytes::Bytes::from(format!(r#"{{"uuid":"{uuid}"}}"#)),
headers: CapturedEventHeaders {
token: Some("token".to_string()),
distinct_id: Some("distinct_id".to_string()),
session_id: None,
timestamp: None,
event: Some("event".to_string()),
uuid: Some(uuid.to_string()),
now: None,
force_disable_person_processing: None,
historical_migration: None,
skip_heatmap_processing: None,
dlq_reason: None,
dlq_step: None,
dlq_timestamp: None,
content_encoding: None,
},
}
}
fn analytics_main() -> PreparedEvent {
prepared(
Address::Lane(PipelineLane::Analytics(AnalyticsLane::Main)),
OrderingGuarantee::PerDistinctId,
)
}
fn outcomes(results: &[SinkResult]) -> Vec<Option<String>> {
results
.iter()
.map(|result| match &result.outcome {
Outcome::Published => None,
Outcome::Failed(err) => Some(format!("{err:?}")),
})
.collect()
}
struct EventInput {
address: Address,
ordering: OrderingGuarantee,
}
struct ExpectedRecord<'a> {
topic: &'a str,
keyed: bool,
}
#[rstest]
#[case::analytics_main(
EventInput { address: Address::Lane(PipelineLane::Analytics(AnalyticsLane::Main)), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "events_plugin_ingestion", keyed: true },
)]
#[case::analytics_overflow_spread(
EventInput { address: Address::Lane(PipelineLane::Analytics(AnalyticsLane::Overflow)), ordering: OrderingGuarantee::None },
ExpectedRecord { topic: "events_plugin_ingestion_overflow", keyed: false },
)]
#[case::analytics_historical(
EventInput { address: Address::Lane(PipelineLane::Analytics(AnalyticsLane::Historical)), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "events_plugin_ingestion_historical", keyed: true },
)]
#[case::ai_main(
EventInput { address: Address::Lane(PipelineLane::Ai(AiLane::Main)), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "ai_events", keyed: true },
)]
#[case::ai_overflow(
EventInput { address: Address::Lane(PipelineLane::Ai(AiLane::Overflow)), ordering: OrderingGuarantee::None },
ExpectedRecord { topic: "ai_events_overflow", keyed: false },
)]
#[case::heatmaps(
EventInput { address: Address::Lane(PipelineLane::Heatmaps(BasicLane::Main)), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "heatmaps", keyed: true },
)]
#[case::warnings(
EventInput { address: Address::Lane(PipelineLane::Warnings(BasicLane::Main)), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "client_ingestion_warning", keyed: true },
)]
#[case::error_tracking(
EventInput { address: Address::Lane(PipelineLane::ErrorTracking(BasicLane::Main)), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "error_tracking_events", keyed: true },
)]
#[case::replay_overflow(
EventInput { address: Address::Lane(PipelineLane::Replay(SessionReplayLane::Overflow)), ordering: OrderingGuarantee::PerSession },
ExpectedRecord { topic: "replay_overflow", keyed: true },
)]
#[case::dlq(
EventInput { address: Address::Dlq, ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "events_plugin_ingestion_dlq", keyed: true },
)]
#[case::custom(
EventInput { address: Address::Custom("admin_topic".to_string()), ordering: OrderingGuarantee::PerDistinctId },
ExpectedRecord { topic: "admin_topic", keyed: true },
)]
#[tokio::test]
async fn prepared_event_lands_on_its_address_topic(
#[case] input: EventInput,
#[case] expected: ExpectedRecord<'_>,
) {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let event = prepared(input.address, input.ordering);
let (payload, headers) = (event.payload.clone(), event.headers.clone());
let results = sink.publish_prepared(vec![event]).await;
assert_eq!(outcomes(&results), vec![None]);
let records = producer.get_records();
assert_eq!(records.len(), 1);
assert_eq!(&*records[0].topic, expected.topic);
assert_eq!(
records[0].key.as_deref(),
expected.keyed.then_some("token:distinct_id")
);
assert_eq!(
records[0].payload, payload,
"the payload passes through unchanged"
);
assert_eq!(
format!("{:?}", records[0].headers),
format!("{headers:?}"),
"the headers pass through unchanged"
);
}
#[tokio::test]
async fn prepared_events_keep_input_order() {
let producer = MockKafkaProducer::new();
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let events: Vec<PreparedEvent> = (0..10).map(|_| analytics_main()).collect();
let payloads: Vec<_> = events.iter().map(|event| event.payload.clone()).collect();
let uuids: Vec<_> = events.iter().map(|event| event.uuid).collect();
let results = sink.publish_prepared(events).await;
let result_uuids: Vec<_> = results.iter().map(|result| result.uuid).collect();
assert_eq!(result_uuids, uuids);
let record_payloads: Vec<_> = producer
.get_records()
.into_iter()
.map(|record| record.payload)
.collect();
assert_eq!(record_payloads, payloads);
}
#[tokio::test]
async fn prepared_enqueue_failure_fails_only_that_event() {
let producer = MockKafkaProducer::new_failing_at(1);
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let results = sink
.publish_prepared(vec![analytics_main(), analytics_main(), analytics_main()])
.await;
assert_eq!(
outcomes(&results),
vec![None, Some("RetryableSinkError".to_string()), None]
);
assert_eq!(producer.get_records().len(), 2);
}
#[tokio::test]
async fn prepared_ack_failure_fails_only_that_event() {
let producer = MockKafkaProducer::new_failing_ack_at(1);
let sink = KafkaSinkBase::with_producer(producer.clone(), test_outputs());
let results = sink
.publish_prepared(vec![analytics_main(), analytics_main(), analytics_main()])
.await;
assert_eq!(
outcomes(&results),
vec![None, Some("RetryableSinkError".to_string()), None]
);
assert_eq!(producer.get_records().len(), 3);
}
#[tokio::test]
async fn prepared_event_publishes_through_its_targets_producer() {
let ingestion = Arc::new(MockKafkaProducer::new());
let ai = Arc::new(MockKafkaProducer::new());
let mut outputs = test_outputs().map_producers(|_| Arc::clone(&ingestion));
outputs.ai_main.producer = Arc::clone(&ai);
let sink = KafkaSinkBase {
outputs: Arc::new(outputs),
replay_envelope_compression: EnvelopeCompression::None,
};
let ai_event = prepared(
Address::Lane(PipelineLane::Ai(AiLane::Main)),
OrderingGuarantee::PerDistinctId,
);
sink.publish_prepared(vec![analytics_main(), ai_event])
.await;
let topics = |producer: &MockKafkaProducer| -> Vec<String> {
producer
.get_records()
.iter()
.map(|record| record.topic.to_string())
.collect()
};
assert_eq!(topics(&ingestion), vec!["events_plugin_ingestion"]);
assert_eq!(topics(&ai), vec!["ai_events"]);
}
}
}