:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use common_kafka::kafka_consumer::{Offset, RecvErr, SingleTopicConsumer};
use futures::FutureExt;
use rayon::prelude::*;
use tracing::{error, info, warn};
use crate::aggregator::Aggregator;
use crate::config::Config;
use crate::metrics_consts::*;
use crate::producer::Producer;
use crate::seen_cache::{utc_day, SeenCache};
use crate::types::{IngestableEvent, PropertyType, TupleKey};
const MAX_BATCH_MESSAGES: usize = 256;
const MAX_BATCH_BYTES: usize = 32 * 1024 * 1024;
#[derive(Clone, Copy, Default)]
pub struct ReductionConfig {
pub max_values_per_key: usize,
pub seen_cache_capacity: usize,
}
/// One worker loop. Each pod runs one worker per input topic.
///
/// At-least-once: on each flush we produce non-transactionally, then on
/// success advance the consumer's stored offsets. Background auto-commit
/// (configured in init_with_defaults) flushes the stored offsets to the
/// broker every 5s. On graceful shutdown we drain one final flush and
/// force a synchronous commit so the next pod resumes from exactly here.
pub async fn worker_loop<E, P, F>(
config: Arc<Config>,
consumer: SingleTopicConsumer,
producer: P,
handle: lifecycle::Handle,
fan_out_fn: F,
worker: &'static str,
reduction: ReductionConfig,
) where
E: IngestableEvent,
P: Producer,
F: Fn(&E) -> Vec<(TupleKey, u64)> + Sync,
{
let _guard = handle.process_scope();
let mut aggregator = Aggregator::new();
let seen_cache = (reduction.seen_cache_capacity > 0)
.then(|| SeenCache::new(reduction.seen_cache_capacity, worker));
// One Offset handle per partition: the latest message we've consumed.
// On successful flush we call .store() on each, which advances the
// consumer's stored offset; auto-commit ships it to the broker.
let mut pending_offsets: HashMap<i32, Offset> = HashMap::new();
let mut flush_timer = tokio::time::interval(Duration::from_secs(config.flush_interval_secs));
flush_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
flush_timer.reset();
loop {
tokio::select! {
_ = handle.shutdown_recv() => {
info!("worker received shutdown; draining final flush");
flush(&mut aggregator, &mut pending_offsets, &producer, FLUSH_REASON_SHUTDOWN, worker, reduction.max_values_per_key, seen_cache.as_ref()).await;
if let Err(e) = consumer.commit() {
warn!(error = %e, "kafka sync commit at shutdown failed; falling back to broker auto-commit");
}
return;
}
_ = flush_timer.tick() => {
handle.report_healthy();
flush(&mut aggregator, &mut pending_offsets, &producer, FLUSH_REASON_TIMER, worker, reduction.max_values_per_key, seen_cache.as_ref()).await;
}
recv = consumer.recv_with(copy_payload) => {
handle.report_healthy();
let (payloads, offsets): (Vec<Vec<u8>>, Vec<Offset>) =
recv_ready_batch(&consumer, recv, worker).into_iter().unzip();
aggregator.merge(aggregate_batch(
&payloads,
|team_id| config.should_process(team_id),
&fan_out_fn,
worker,
));
for offset in offsets {
pending_offsets.insert(offset.partition(), offset);
}
if aggregator.len() >= config.max_buffered_tuples {
flush(
&mut aggregator,
&mut pending_offsets,
&producer,
FLUSH_REASON_BACKPRESSURE,
worker,
reduction.max_values_per_key,
seen_cache.as_ref(),
).await;
}
}
}
}
}
fn copy_payload(payload: &[u8]) -> Result<Vec<u8>, serde_json::Error> {
Ok(payload.to_vec())
}
fn recv_ready_batch(
consumer: &SingleTopicConsumer,
first: Result<(Vec<u8>, Offset), RecvErr>,
worker: &'static str,
) -> Vec<(Vec<u8>, Offset)> {
drain_ready(
first,
|| consumer.recv_with(copy_payload).now_or_never(),
worker,
)
}
fn drain_ready<T>(
first: Result<(Vec<u8>, T), RecvErr>,
mut next_ready: impl FnMut() -> Option<Result<(Vec<u8>, T), RecvErr>>,
worker: &'static str,
) -> Vec<(Vec<u8>, T)> {
let mut batch = Vec::new();
let mut received = 0;
let mut batch_bytes = 0;
let mut next = Some(first);
while let Some(recv) = next {
received += 1;
match recv {
Ok((payload, offset)) => {
batch_bytes += payload.len();
batch.push((payload, offset));
}
Err(RecvErr::Empty) | Err(RecvErr::Serde(_)) => {
// SingleTopicConsumer auto-stores poison-pill offsets.
}
Err(RecvErr::Kafka(e)) => {
metrics::counter!(KAFKA_RECV_ERRORS, "worker" => worker).increment(1);
warn!(error = %e, "kafka recv error");
break;
}
}
next = if received < MAX_BATCH_MESSAGES && batch_bytes < MAX_BATCH_BYTES {
next_ready()
} else {
None
};
}
batch
}
fn aggregate_batch<E, F>(
payloads: &[Vec<u8>],
should_process: impl Fn(i64) -> bool + Sync,
fan_out_fn: &F,
worker: &'static str,
) -> Aggregator
where
E: IngestableEvent,
F: Fn(&E) -> Vec<(TupleKey, u64)> + Sync,
{
payloads
.par_iter()
.fold(Aggregator::new, |mut aggregator, payload| {
let Ok(event) = E::decode(payload) else {
return aggregator;
};
metrics::counter!(EVENTS_RECEIVED, "worker" => worker).increment(1);
if !should_process(event.team_id()) {
metrics::counter!(EVENTS_FILTERED, "worker" => worker).increment(1);
return aggregator;
}
let tuples = fan_out_fn(&event);
metrics::counter!(TUPLES_AGGREGATED, "worker" => worker).increment(tuples.len() as u64);
for (tuple, count) in tuples {
aggregator.add(tuple, count);
}
aggregator
})
.reduce(Aggregator::new, |mut left, right| {
left.merge(right);
left
})
}
/// Produce the aggregated counts to the output topic, then advance the
/// consumer's stored offsets so they get auto-committed. On produce failure,
/// the snapshot is merged back into the aggregator and offsets are left
/// untouched so the next flush retries the same input range.
pub(crate) async fn flush<P: Producer>(
aggregator: &mut Aggregator,
pending_offsets: &mut HashMap<i32, Offset>,
producer: &P,
reason: &'static str,
worker: &'static str,
max_values_per_key: usize,
seen_cache: Option<&SeenCache>,
) {
if aggregator.is_empty() && pending_offsets.is_empty() {
return;
}
if let Some(cache) = seen_cache {
cache.roll(utc_day());
}
let mut snapshot: Vec<(TupleKey, u64)> = aggregator.drain().into_iter().collect();
if max_values_per_key > 0 {
snapshot = cap_top_k(snapshot, max_values_per_key, worker);
}
let to_emit: Vec<(TupleKey, u64)> = match seen_cache {
Some(cache) => snapshot
.into_iter()
.filter(|(tuple, _)| !cache.seen(tuple))
.collect(),
None => snapshot,
};
metrics::counter!(FLUSH_TOTAL, "reason" => reason, "worker" => worker).increment(1);
metrics::histogram!(FLUSH_TUPLES, "worker" => worker).record(to_emit.len() as f64);
for (_, count) in &to_emit {
metrics::histogram!(FLUSH_TUPLE_COUNT, "worker" => worker).record(*count as f64);
}
if !to_emit.is_empty() {
if let Err(e) = producer.produce(to_emit.clone()).await {
metrics::counter!(PRODUCER_FLUSH_FAILED, "worker" => worker).increment(1);
error!(error = %e, "produce failed; restoring counts, retrying next flush");
for (tuple, count) in to_emit {
aggregator.add(tuple, count);
}
return;
}
if let Some(cache) = seen_cache {
for (tuple, _) in &to_emit {
cache.insert(tuple);
}
}
}
// Produce succeeded; advance the stored offset for each partition we
// consumed from. Background auto-commit will ship these to the broker
// within ~5s; shutdown forces a sync commit.
for (_partition, offset) in pending_offsets.drain() {
if let Err(e) = offset.store() {
metrics::counter!(OFFSET_STORE_FAILED, "worker" => worker).increment(1);
warn!(error = %e, "failed to store offset; auto-commit will be a no-op for this partition until the next successful flush");
}
}
}
/// Cap each (team, type, key) to its `k` highest-count values, dropping
/// the rest. Keys with `<= k` distinct values are untouched, so low-cardinality
/// keys keep everything; only high-cardinality keys lose their long tail.
type CapGroup = (i64, PropertyType, String);
fn cap_top_k(
snapshot: Vec<(TupleKey, u64)>,
k: usize,
worker: &'static str,
) -> Vec<(TupleKey, u64)> {
let mut by_key: HashMap<CapGroup, Vec<(TupleKey, u64)>> = HashMap::new();
for entry in snapshot {
let group = (
entry.0.team_id,
entry.0.property_type,
entry.0.property_key.clone(),
);
by_key.entry(group).or_default().push(entry);
}
let mut out = Vec::new();
for (_, mut values) in by_key {
if values.len() > k {
values.select_nth_unstable_by(k, |a, b| b.1.cmp(&a.1));
let dropped = values.len() - k;
values.truncate(k);
metrics::counter!(TOP_K_DROPPED, "worker" => worker).increment(dropped as u64);
}
out.extend(values);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::aggregator::Aggregator;
use crate::config::{ExcludedPropertyKeys, LengthCaps};
use crate::fan_out::fan_out;
use crate::producer::ProduceError;
use crate::types::{Event, PropertyType, TupleKey};
use proptest::prelude::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
/// Mock producer that records each call and can be configured to fail on
/// specific call indices.
struct MockProducer {
calls: AtomicUsize,
fail_on: Mutex<Vec<usize>>,
seen_items: Mutex<Vec<Vec<(TupleKey, u64)>>>,
}
impl MockProducer {
fn new() -> Self {
Self {
calls: AtomicUsize::new(0),
fail_on: Mutex::new(Vec::new()),
seen_items: Mutex::new(Vec::new()),
}
}
fn fail_on(self, call_index: usize) -> Self {
self.fail_on.lock().unwrap().push(call_index);
self
}
fn call_count(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
fn last_items(&self) -> Vec<(TupleKey, u64)> {
self.seen_items
.lock()
.unwrap()
.last()
.cloned()
.unwrap_or_default()
}
fn successful_batches(&self) -> Vec<Vec<(TupleKey, u64)>> {
let items = self.seen_items.lock().unwrap();
let fail = self.fail_on.lock().unwrap();
items
.iter()
.enumerate()
.filter(|(i, _)| !fail.contains(&(i + 1)))
.map(|(_, batch)| batch.clone())
.collect()
}
}
#[async_trait::async_trait]
impl Producer for MockProducer {
async fn produce(&self, items: Vec<(TupleKey, u64)>) -> Result<(), ProduceError> {
let n = self.calls.fetch_add(1, Ordering::SeqCst) + 1;
self.seen_items.lock().unwrap().push(items.clone());
if self.fail_on.lock().unwrap().contains(&n) {
let total = items.len().max(1);
return Err(ProduceError::PartialFailure {
failed: total,
total,
});
}
Ok(())
}
}
fn tuple(team: i64, key: &str, value: &str) -> TupleKey {
TupleKey {
team_id: team,
property_type: PropertyType::Event,
property_key: key.to_string(),
property_value: value.to_string(),
}
}
fn populate(agg: &mut Aggregator, count: u64) {
for i in 0..count {
agg.add(tuple(2, "k", &format!("v{i}")), 1);
}
}
#[test]
fn aggregate_batch_counts_every_kept_event_once() {
let payloads: Vec<Vec<u8>> = (0..2_000)
.map(|i| {
serde_json::to_vec(&serde_json::json!({
"team_id": if i % 10 == 0 { 1 } else { 2 },
"properties": r#"{"$feature/a": true, "$browser": "Chrome"}"#,
}))
.unwrap()
})
.chain([b"not json".to_vec()])
.collect();
let excluded = ExcludedPropertyKeys::default();
let caps = LengthCaps {
max_property_key_len: 400,
max_property_value_len: 10_000,
};
let mut batch = aggregate_batch(
&payloads,
|team_id| team_id == 2,
&|event: &Event| fan_out(event, &excluded, caps),
"test",
);
let expected = HashMap::from([
(tuple(2, "$feature/a", "true"), 1_800),
(tuple(2, "$browser", "Chrome"), 1_800),
]);
assert_eq!(batch.drain(), expected);
}
#[test]
fn drain_ready_counts_empty_messages_toward_the_batch_limit() {
let mut ready = std::iter::repeat_with(|| Err(RecvErr::Empty))
.take(2 * MAX_BATCH_MESSAGES)
.chain((0..MAX_BATCH_MESSAGES).map(|offset| Ok((b"{}".to_vec(), offset))));
let batch = drain_ready(Ok((b"{}".to_vec(), 0)), || ready.next(), "test");
assert_eq!(batch.len(), 1);
assert_eq!(ready.count(), 2 * MAX_BATCH_MESSAGES + 1);
}
#[test]
fn drain_ready_stops_once_payloads_reach_the_byte_budget() {
let payload = || vec![0; MAX_BATCH_BYTES / 4];
let mut ready = (1..10).map(|offset| Ok((payload(), offset)));
let batch = drain_ready(Ok((payload(), 0)), || ready.next(), "test");
assert_eq!(batch.len(), 4);
}
#[test]
fn cap_top_k_keeps_keys_under_the_cap() {
let snap = vec![
(tuple(2, "$browser", "Chrome"), 5),
(tuple(2, "$browser", "Firefox"), 1),
(tuple(2, "$browser", "Safari"), 1),
];
let out = cap_top_k(snap, 10, "test");
assert_eq!(
out.len(),
3,
"a key with fewer than k values keeps all of them"
);
}
#[test]
fn cap_top_k_drops_tail_of_high_card_key() {
let snap = vec![
(tuple(2, "$insert_id", "a"), 1),
(tuple(2, "$insert_id", "b"), 9),
(tuple(2, "$insert_id", "c"), 5),
(tuple(2, "$insert_id", "d"), 1),
];
let out = cap_top_k(snap, 2, "test");
assert_eq!(out.len(), 2);
let counts: Vec<u64> = out.iter().map(|(_, c)| *c).collect();
assert!(
counts.contains(&9) && counts.contains(&5),
"keeps the two highest counts"
);
}
#[test]
fn cap_top_k_caps_each_key_independently() {
let snap = vec![
(tuple(2, "$browser", "Chrome"), 3),
(tuple(2, "$insert_id", "a"), 1),
(tuple(2, "$insert_id", "b"), 2),
(tuple(2, "$insert_id", "c"), 3),
];
let out = cap_top_k(snap, 2, "test");
let browser = out
.iter()
.filter(|(t, _)| t.property_key == "$browser")
.count();
let insert = out
.iter()
.filter(|(t, _)| t.property_key == "$insert_id")
.count();
assert_eq!(browser, 1, "low-card key untouched");
assert_eq!(insert, 2, "high-card key capped to k");
}
#[tokio::test]
async fn successful_flush_drains_aggregator() {
let mut agg = Aggregator::new();
populate(&mut agg, 5);
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new();
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
assert!(agg.is_empty());
assert_eq!(producer.call_count(), 1);
assert_eq!(producer.last_items().len(), 5);
}
#[tokio::test]
async fn failed_flush_restores_counts() {
let mut agg = Aggregator::new();
populate(&mut agg, 3);
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new().fail_on(1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
assert_eq!(
agg.len(),
3,
"aggregator must restore drained counts on produce failure"
);
}
#[tokio::test]
async fn failed_then_successful_flush_eventually_clears_state() {
let mut agg = Aggregator::new();
populate(&mut agg, 4);
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new().fail_on(1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
assert!(!agg.is_empty());
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
assert!(agg.is_empty());
assert_eq!(producer.call_count(), 2);
}
#[tokio::test]
async fn empty_aggregator_empty_offsets_is_noop() {
let mut agg = Aggregator::new();
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new();
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
assert_eq!(producer.call_count(), 0);
}
#[tokio::test]
async fn restored_counts_merge_with_new_counts_in_next_window() {
let mut agg = Aggregator::new();
agg.add(tuple(2, "k1", "v1"), 1);
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new().fail_on(1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
agg.add(tuple(2, "k1", "v1"), 1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
let batch = producer.last_items();
assert_eq!(batch.len(), 1);
assert_eq!(batch[0].1, 2);
}
#[tokio::test]
async fn repeated_failure_holds_all_state_indefinitely() {
let mut agg = Aggregator::new();
agg.add(tuple(2, "k1", "v1"), 1);
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new().fail_on(1).fail_on(2).fail_on(3);
for _ in 0..3 {
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
)
.await;
}
assert_eq!(agg.len(), 1);
assert_eq!(producer.call_count(), 3);
}
#[tokio::test]
async fn seen_cache_suppresses_already_emitted_tuples() {
let cache = SeenCache::new(1000, "test");
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new();
let mut agg = Aggregator::new();
agg.add(tuple(2, "k", "v"), 1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
Some(&cache),
)
.await;
assert_eq!(producer.last_items().len(), 1, "first sight is emitted");
agg.add(tuple(2, "k", "v"), 1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
Some(&cache),
)
.await;
assert_eq!(
producer.call_count(),
1,
"an already-cached tuple is suppressed, so there is no second produce"
);
assert!(
agg.is_empty(),
"suppressed tuples still drain the aggregator"
);
}
#[tokio::test]
async fn seen_cache_reemits_after_produce_failure() {
let cache = SeenCache::new(1000, "test");
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new().fail_on(1);
let mut agg = Aggregator::new();
agg.add(tuple(2, "k", "v"), 1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
Some(&cache),
)
.await;
assert_eq!(agg.len(), 1, "failed produce restores the tuple for retry");
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
Some(&cache),
)
.await;
assert_eq!(producer.call_count(), 2);
assert_eq!(
producer.last_items().len(),
1,
"a tuple forgotten on produce failure is re-emitted, not lost to the cache"
);
}
#[tokio::test]
async fn seen_cache_emits_new_values_and_suppresses_repeats() {
let cache = SeenCache::new(1000, "test");
let mut pending: HashMap<i32, Offset> = HashMap::new();
let producer = MockProducer::new();
let mut agg = Aggregator::new();
agg.add(tuple(2, "k", "a"), 1);
agg.add(tuple(2, "k", "b"), 1);
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
Some(&cache),
)
.await;
assert_eq!(
producer.last_items().len(),
2,
"distinct new tuples all emit"
);
agg.add(tuple(2, "k", "a"), 1); // already seen
agg.add(tuple(2, "k", "c"), 1); // new
flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
Some(&cache),
)
.await;
let batch = producer.last_items();
assert_eq!(
batch.len(),
1,
"only the new value emits; the repeat is suppressed"
);
assert_eq!(batch[0].0.property_value, "c");
}
fn arb_property_type() -> impl Strategy<Value = PropertyType> {
prop_oneof![
Just(PropertyType::Event),
Just(PropertyType::Person),
(0u8..=4).prop_map(PropertyType::Group),
]
}
prop_compose! {
fn arb_tuple()(
team_id in -3i64..=3,
property_type in arb_property_type(),
property_key in "[a-c]{1,2}",
property_value in "[x-z]{1,2}",
) -> TupleKey {
TupleKey { team_id, property_type, property_key, property_value }
}
}
#[derive(Debug, Clone)]
enum WorkerOp {
Add(TupleKey, u64),
Flush,
}
fn arb_worker_op() -> impl Strategy<Value = WorkerOp> {
prop_oneof![
// 4:1 weighting so the aggregator actually accumulates between flushes
// instead of every other op being an immediate drain.
4 => (arb_tuple(), 1u64..100).prop_map(|(t, n)| WorkerOp::Add(t, n)),
1 => Just(WorkerOp::Flush),
]
}
proptest! {
#[test]
fn flush_conserves_counts(
ops in prop::collection::vec(arb_worker_op(), 0..60),
fail_indices in prop::collection::vec(1usize..30, 0..15),
) {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let mut agg = Aggregator::new();
let mut pending: HashMap<i32, Offset> = HashMap::new();
let mut producer = MockProducer::new();
for idx in &fail_indices {
producer = producer.fail_on(*idx);
}
let mut recorded: HashMap<TupleKey, u64> = HashMap::new();
for op in &ops {
match op {
WorkerOp::Add(t, n) => {
agg.add(t.clone(), *n);
*recorded.entry(t.clone()).or_insert(0) += *n;
}
WorkerOp::Flush => {
runtime.block_on(flush(
&mut agg,
&mut pending,
&producer,
FLUSH_REASON_TIMER,
"test",
0,
None,
));
}
}
}
let mut accounted: HashMap<TupleKey, u64> = HashMap::new();
for (t, n) in agg.drain() {
*accounted.entry(t).or_insert(0) += n;
}
for batch in producer.successful_batches() {
for (t, n) in batch {
*accounted.entry(t).or_insert(0) += n;
}
}
prop_assert_eq!(recorded, accounted);
}
}
}