Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(propvals): process each event batch on all cores (#124337)

Andy Zhao40d1ae7e05d3
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 33bc36d55957..bcd207c538d8 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -10074,10 +10074,12 @@ dependencies = [
"common-kafka",
"common-liveness",
"envconfig",
+ "futures",
"lifecycle",
"metrics",
"proptest",
"quick_cache",
+ "rayon",
"rdkafka",
"serde",
"serde_json",
diff --git a/rust/property-vals-rs/Cargo.toml b/rust/property-vals-rs/Cargo.toml
index 424a4f6fa096..3b9656770323 100644
--- a/rust/property-vals-rs/Cargo.toml
+++ b/rust/property-vals-rs/Cargo.toml
@@ -12,9 +12,11 @@ common-continuous-profiling = { path = "../common/continuous_profiling" }
common-kafka = { path = "../common/kafka" }
common-liveness = { path = "../common/liveness" }
envconfig = { workspace = true }
+futures = { workspace = true }
lifecycle = { path = "../common/lifecycle" }
metrics = { workspace = true }
quick_cache = { workspace = true }
+rayon = "1.11.0"
rdkafka = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
diff --git a/rust/property-vals-rs/src/aggregator.rs b/rust/property-vals-rs/src/aggregator.rs
index e3e55569f976..4cb0468ec71a 100644
--- a/rust/property-vals-rs/src/aggregator.rs
+++ b/rust/property-vals-rs/src/aggregator.rs
@@ -17,6 +17,12 @@ impl Aggregator {
*self.counts.entry(tuple).or_insert(0) += count;
}
+ pub fn merge(&mut self, other: Aggregator) {
+ for (tuple, count) in other.counts {
+ self.add(tuple, count);
+ }
+ }
+
pub fn len(&self) -> usize {
self.counts.len()
}
diff --git a/rust/property-vals-rs/src/worker.rs b/rust/property-vals-rs/src/worker.rs
index 279815a262e8..271d06926242 100644
--- a/rust/property-vals-rs/src/worker.rs
+++ b/rust/property-vals-rs/src/worker.rs
@@ -3,6 +3,8 @@ use std::sync::Arc;
use std::time::Duration;
use common_kafka::kafka_consumer::{Offset, RecvErr, SingleTopicConsumer};
+use futures::FutureExt;
+use rayon::prelude::*;
use tracing::{error, info, warn};
use crate::aggregator::Aggregator;
@@ -12,6 +14,9 @@ use crate::producer::Producer;
use crate::seen_cache::{utc_day, SeenCache};
use crate::types::{IngestableEvent, PropertyType, TupleKey};
+const MAX_BATCH_MESSAGES: usize = 256;
+const MAX_BATCH_BYTES: usize = 32 * 1024 * 1024;
+
#[derive(Clone, Copy, Default)]
pub struct ReductionConfig {
pub max_values_per_key: usize,
@@ -36,7 +41,7 @@ pub async fn worker_loop<E, P, F>(
) where
E: IngestableEvent,
P: Producer,
- F: Fn(&E) -> Vec<(TupleKey, u64)>,
+ F: Fn(&E) -> Vec<(TupleKey, u64)> + Sync,
{
let _guard = handle.process_scope();
@@ -66,49 +71,120 @@ pub async fn worker_loop<E, P, F>(
handle.report_healthy();
flush(&mut aggregator, &mut pending_offsets, &producer, FLUSH_REASON_TIMER, worker, reduction.max_values_per_key, seen_cache.as_ref()).await;
}
- recv = consumer.recv_with(E::decode) => {
+ recv = consumer.recv_with(copy_payload) => {
handle.report_healthy();
- match recv {
- Ok((event, offset)) => {
- metrics::counter!(EVENTS_RECEIVED, "worker" => worker).increment(1);
-
- if config.should_process(event.team_id()) {
- let tuples = fan_out_fn(&event);
- metrics::counter!(TUPLES_AGGREGATED, "worker" => worker).increment(tuples.len() as u64);
- for (t, count) in tuples {
- aggregator.add(t, count);
- }
- } else {
- metrics::counter!(EVENTS_FILTERED, "worker" => worker).increment(1);
- }
-
- pending_offsets.insert(offset.partition(), offset);
-
- if aggregator.len() >= config.max_buffered_tuples {
- flush(
- &mut aggregator,
- &mut pending_offsets,
- &producer,
- FLUSH_REASON_BACKPRESSURE,
- worker,
- reduction.max_values_per_key,
- seen_cache.as_ref(),
- ).await;
- }
- }
- Err(RecvErr::Empty) | Err(RecvErr::Serde(_)) => {
- // SingleTopicConsumer auto-stores poison-pill offsets.
- }
- Err(RecvErr::Kafka(e)) => {
- metrics::counter!(KAFKA_RECV_ERRORS, "worker" => worker).increment(1);
- warn!(error = %e, "kafka recv error");
- }
+ let (payloads, offsets): (Vec<Vec<u8>>, Vec<Offset>) =
+ recv_ready_batch(&consumer, recv, worker).into_iter().unzip();
+ aggregator.merge(aggregate_batch(
+ &payloads,
+ |team_id| config.should_process(team_id),
+ &fan_out_fn,
+ worker,
+ ));
+ for offset in offsets {
+ pending_offsets.insert(offset.partition(), offset);
+ }
+
+ if aggregator.len() >= config.max_buffered_tuples {
+ flush(
+ &mut aggregator,
+ &mut pending_offsets,
+ &producer,
+ FLUSH_REASON_BACKPRESSURE,
+ worker,
+ reduction.max_values_per_key,
+ seen_cache.as_ref(),
+ ).await;
}
}
}
}
}
+fn copy_payload(payload: &[u8]) -> Result<Vec<u8>, serde_json::Error> {
+ Ok(payload.to_vec())
+}
+
+fn recv_ready_batch(
+ consumer: &SingleTopicConsumer,
+ first: Result<(Vec<u8>, Offset), RecvErr>,
+ worker: &'static str,
+) -> Vec<(Vec<u8>, Offset)> {
+ drain_ready(
+ first,
+ || consumer.recv_with(copy_payload).now_or_never(),
+ worker,
+ )
+}
+
+fn drain_ready<T>(
+ first: Result<(Vec<u8>, T), RecvErr>,
+ mut next_ready: impl FnMut() -> Option<Result<(Vec<u8>, T), RecvErr>>,
+ worker: &'static str,
+) -> Vec<(Vec<u8>, T)> {
+ let mut batch = Vec::new();
+ let mut received = 0;
+ let mut batch_bytes = 0;
+ let mut next = Some(first);
+ while let Some(recv) = next {
+ received += 1;
+ match recv {
+ Ok((payload, offset)) => {
+ batch_bytes += payload.len();
+ batch.push((payload, offset));
+ }
+ Err(RecvErr::Empty) | Err(RecvErr::Serde(_)) => {
+ // SingleTopicConsumer auto-stores poison-pill offsets.
+ }
+ Err(RecvErr::Kafka(e)) => {
+ metrics::counter!(KAFKA_RECV_ERRORS, "worker" => worker).increment(1);
+ warn!(error = %e, "kafka recv error");
+ break;
+ }
+ }
+ next = if received < MAX_BATCH_MESSAGES && batch_bytes < MAX_BATCH_BYTES {
+ next_ready()
+ } else {
+ None
+ };
+ }
+ batch
+}
+
+fn aggregate_batch<E, F>(
+ payloads: &[Vec<u8>],
+ should_process: impl Fn(i64) -> bool + Sync,
+ fan_out_fn: &F,
+ worker: &'static str,
+) -> Aggregator
+where
+ E: IngestableEvent,
+ F: Fn(&E) -> Vec<(TupleKey, u64)> + Sync,
+{
+ payloads
+ .par_iter()
+ .fold(Aggregator::new, |mut aggregator, payload| {
+ let Ok(event) = E::decode(payload) else {
+ return aggregator;
+ };
+ metrics::counter!(EVENTS_RECEIVED, "worker" => worker).increment(1);
+ if !should_process(event.team_id()) {
+ metrics::counter!(EVENTS_FILTERED, "worker" => worker).increment(1);
+ return aggregator;
+ }
+ let tuples = fan_out_fn(&event);
+ metrics::counter!(TUPLES_AGGREGATED, "worker" => worker).increment(tuples.len() as u64);
+ for (tuple, count) in tuples {
+ aggregator.add(tuple, count);
+ }
+ aggregator
+ })
+ .reduce(Aggregator::new, |mut left, right| {
+ left.merge(right);
+ left
+ })
+}
+
/// Produce the aggregated counts to the output topic, then advance the
/// consumer's stored offsets so they get auto-committed. On produce failure,
/// the snapshot is merged back into the aggregator and offsets are left
@@ -214,8 +290,10 @@ fn cap_top_k(
mod tests {
use super::*;
use crate::aggregator::Aggregator;
+ use crate::config::{ExcludedPropertyKeys, LengthCaps};
+ use crate::fan_out::fan_out;
use crate::producer::ProduceError;
- use crate::types::{PropertyType, TupleKey};
+ use crate::types::{Event, PropertyType, TupleKey};
use proptest::prelude::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
@@ -294,6 +372,60 @@ mod tests {
}
}
+ #[test]
+ fn aggregate_batch_counts_every_kept_event_once() {
+ let payloads: Vec<Vec<u8>> = (0..2_000)
+ .map(|i| {
+ serde_json::to_vec(&serde_json::json!({
+ "team_id": if i % 10 == 0 { 1 } else { 2 },
+ "properties": r#"{"$feature/a": true, "$browser": "Chrome"}"#,
+ }))
+ .unwrap()
+ })
+ .chain([b"not json".to_vec()])
+ .collect();
+ let excluded = ExcludedPropertyKeys::default();
+ let caps = LengthCaps {
+ max_property_key_len: 400,
+ max_property_value_len: 10_000,
+ };
+
+ let mut batch = aggregate_batch(
+ &payloads,
+ |team_id| team_id == 2,
+ &|event: &Event| fan_out(event, &excluded, caps),
+ "test",
+ );
+
+ let expected = HashMap::from([
+ (tuple(2, "$feature/a", "true"), 1_800),
+ (tuple(2, "$browser", "Chrome"), 1_800),
+ ]);
+ assert_eq!(batch.drain(), expected);
+ }
+
+ #[test]
+ fn drain_ready_counts_empty_messages_toward_the_batch_limit() {
+ let mut ready = std::iter::repeat_with(|| Err(RecvErr::Empty))
+ .take(2 * MAX_BATCH_MESSAGES)
+ .chain((0..MAX_BATCH_MESSAGES).map(|offset| Ok((b"{}".to_vec(), offset))));
+
+ let batch = drain_ready(Ok((b"{}".to_vec(), 0)), || ready.next(), "test");
+
+ assert_eq!(batch.len(), 1);
+ assert_eq!(ready.count(), 2 * MAX_BATCH_MESSAGES + 1);
+ }
+
+ #[test]
+ fn drain_ready_stops_once_payloads_reach_the_byte_budget() {
+ let payload = || vec![0; MAX_BATCH_BYTES / 4];
+ let mut ready = (1..10).map(|offset| Ok((payload(), offset)));
+
+ let batch = drain_ready(Ok((payload(), 0)), || ready.next(), "test");
+
+ assert_eq!(batch.len(), 4);
+ }
+
#[test]
fn cap_top_k_keeps_keys_under_the_cap() {
let snap = vec![