:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(propvals): process each event batch on all cores (#124337)
Andy Zhao40d1ae7e05d3
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 33bc36d55957..bcd207c538d8 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -10074,10 +10074,12 @@ dependencies = ["common-kafka","common-liveness","envconfig",+ "futures","lifecycle","metrics","proptest","quick_cache",+ "rayon","rdkafka","serde","serde_json",diff --git a/rust/property-vals-rs/Cargo.toml b/rust/property-vals-rs/Cargo.tomlindex 424a4f6fa096..3b9656770323 100644--- a/rust/property-vals-rs/Cargo.toml+++ b/rust/property-vals-rs/Cargo.toml@@ -12,9 +12,11 @@ common-continuous-profiling = { path = "../common/continuous_profiling" }common-kafka = { path = "../common/kafka" }common-liveness = { path = "../common/liveness" }envconfig = { workspace = true }+futures = { workspace = true }lifecycle = { path = "../common/lifecycle" }metrics = { workspace = true }quick_cache = { workspace = true }+rayon = "1.11.0"rdkafka = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }diff --git a/rust/property-vals-rs/src/aggregator.rs b/rust/property-vals-rs/src/aggregator.rsindex e3e55569f976..4cb0468ec71a 100644--- a/rust/property-vals-rs/src/aggregator.rs+++ b/rust/property-vals-rs/src/aggregator.rs@@ -17,6 +17,12 @@ impl Aggregator {*self.counts.entry(tuple).or_insert(0) += count;}+ pub fn merge(&mut self, other: Aggregator) {+ for (tuple, count) in other.counts {+ self.add(tuple, count);+ }+ }+pub fn len(&self) -> usize {self.counts.len()}diff --git a/rust/property-vals-rs/src/worker.rs b/rust/property-vals-rs/src/worker.rsindex 279815a262e8..271d06926242 100644--- a/rust/property-vals-rs/src/worker.rs+++ b/rust/property-vals-rs/src/worker.rs@@ -3,6 +3,8 @@ use std::sync::Arc;use std::time::Duration;use common_kafka::kafka_consumer::{Offset, RecvErr, SingleTopicConsumer};+use futures::FutureExt;+use rayon::prelude::*;use tracing::{error, info, warn};use crate::aggregator::Aggregator;@@ -12,6 +14,9 @@ use crate::producer::Producer;use crate::seen_cache::{utc_day, SeenCache};use crate::types::{IngestableEvent, PropertyType, TupleKey};+const MAX_BATCH_MESSAGES: usize = 256;+const MAX_BATCH_BYTES: usize = 32 * 1024 * 1024;+#[derive(Clone, Copy, Default)]pub struct ReductionConfig {pub max_values_per_key: usize,@@ -36,7 +41,7 @@ pub async fn worker_loop<E, P, F>() whereE: IngestableEvent,P: Producer,- F: Fn(&E) -> Vec<(TupleKey, u64)>,+ F: Fn(&E) -> Vec<(TupleKey, u64)> + Sync,{let _guard = handle.process_scope();@@ -66,49 +71,120 @@ pub async fn worker_loop<E, P, F>(handle.report_healthy();flush(&mut aggregator, &mut pending_offsets, &producer, FLUSH_REASON_TIMER, worker, reduction.max_values_per_key, seen_cache.as_ref()).await;}- recv = consumer.recv_with(E::decode) => {+ recv = consumer.recv_with(copy_payload) => {handle.report_healthy();- match recv {- Ok((event, offset)) => {- metrics::counter!(EVENTS_RECEIVED, "worker" => worker).increment(1);-- if config.should_process(event.team_id()) {- let tuples = fan_out_fn(&event);- metrics::counter!(TUPLES_AGGREGATED, "worker" => worker).increment(tuples.len() as u64);- for (t, count) in tuples {- aggregator.add(t, count);- }- } else {- metrics::counter!(EVENTS_FILTERED, "worker" => worker).increment(1);- }-- pending_offsets.insert(offset.partition(), offset);-- if aggregator.len() >= config.max_buffered_tuples {- flush(- &mut aggregator,- &mut pending_offsets,- &producer,- FLUSH_REASON_BACKPRESSURE,- worker,- reduction.max_values_per_key,- seen_cache.as_ref(),- ).await;- }- }- Err(RecvErr::Empty) | Err(RecvErr::Serde(_)) => {- // SingleTopicConsumer auto-stores poison-pill offsets.- }- Err(RecvErr::Kafka(e)) => {- metrics::counter!(KAFKA_RECV_ERRORS, "worker" => worker).increment(1);- warn!(error = %e, "kafka recv error");- }+ let (payloads, offsets): (Vec<Vec<u8>>, Vec<Offset>) =+ recv_ready_batch(&consumer, recv, worker).into_iter().unzip();+ aggregator.merge(aggregate_batch(+ &payloads,+ |team_id| config.should_process(team_id),+ &fan_out_fn,+ worker,+ ));+ for offset in offsets {+ pending_offsets.insert(offset.partition(), offset);+ }++ if aggregator.len() >= config.max_buffered_tuples {+ flush(+ &mut aggregator,+ &mut pending_offsets,+ &producer,+ FLUSH_REASON_BACKPRESSURE,+ worker,+ reduction.max_values_per_key,+ seen_cache.as_ref(),+ ).await;}}}}}+fn copy_payload(payload: &[u8]) -> Result<Vec<u8>, serde_json::Error> {+ Ok(payload.to_vec())+}++fn recv_ready_batch(+ consumer: &SingleTopicConsumer,+ first: Result<(Vec<u8>, Offset), RecvErr>,+ worker: &'static str,+) -> Vec<(Vec<u8>, Offset)> {+ drain_ready(+ first,+ || consumer.recv_with(copy_payload).now_or_never(),+ worker,+ )+}++fn drain_ready<T>(+ first: Result<(Vec<u8>, T), RecvErr>,+ mut next_ready: impl FnMut() -> Option<Result<(Vec<u8>, T), RecvErr>>,+ worker: &'static str,+) -> Vec<(Vec<u8>, T)> {+ let mut batch = Vec::new();+ let mut received = 0;+ let mut batch_bytes = 0;+ let mut next = Some(first);+ while let Some(recv) = next {+ received += 1;+ match recv {+ Ok((payload, offset)) => {+ batch_bytes += payload.len();+ batch.push((payload, offset));+ }+ Err(RecvErr::Empty) | Err(RecvErr::Serde(_)) => {+ // SingleTopicConsumer auto-stores poison-pill offsets.+ }+ Err(RecvErr::Kafka(e)) => {+ metrics::counter!(KAFKA_RECV_ERRORS, "worker" => worker).increment(1);+ warn!(error = %e, "kafka recv error");+ break;+ }+ }+ next = if received < MAX_BATCH_MESSAGES && batch_bytes < MAX_BATCH_BYTES {+ next_ready()+ } else {+ None+ };+ }+ batch+}++fn aggregate_batch<E, F>(+ payloads: &[Vec<u8>],+ should_process: impl Fn(i64) -> bool + Sync,+ fan_out_fn: &F,+ worker: &'static str,+) -> Aggregator+where+ E: IngestableEvent,+ F: Fn(&E) -> Vec<(TupleKey, u64)> + Sync,+{+ payloads+ .par_iter()+ .fold(Aggregator::new, |mut aggregator, payload| {+ let Ok(event) = E::decode(payload) else {+ return aggregator;+ };+ metrics::counter!(EVENTS_RECEIVED, "worker" => worker).increment(1);+ if !should_process(event.team_id()) {+ metrics::counter!(EVENTS_FILTERED, "worker" => worker).increment(1);+ return aggregator;+ }+ let tuples = fan_out_fn(&event);+ metrics::counter!(TUPLES_AGGREGATED, "worker" => worker).increment(tuples.len() as u64);+ for (tuple, count) in tuples {+ aggregator.add(tuple, count);+ }+ aggregator+ })+ .reduce(Aggregator::new, |mut left, right| {+ left.merge(right);+ left+ })+}+/// Produce the aggregated counts to the output topic, then advance the/// consumer's stored offsets so they get auto-committed. On produce failure,/// the snapshot is merged back into the aggregator and offsets are left@@ -214,8 +290,10 @@ fn cap_top_k(mod tests {use super::*;use crate::aggregator::Aggregator;+ use crate::config::{ExcludedPropertyKeys, LengthCaps};+ use crate::fan_out::fan_out;use crate::producer::ProduceError;- use crate::types::{PropertyType, TupleKey};+ use crate::types::{Event, PropertyType, TupleKey};use proptest::prelude::*;use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::Mutex;@@ -294,6 +372,60 @@ mod tests {}}+ #[test]+ fn aggregate_batch_counts_every_kept_event_once() {+ let payloads: Vec<Vec<u8>> = (0..2_000)+ .map(|i| {+ serde_json::to_vec(&serde_json::json!({+ "team_id": if i % 10 == 0 { 1 } else { 2 },+ "properties": r#"{"$feature/a": true, "$browser": "Chrome"}"#,+ }))+ .unwrap()+ })+ .chain([b"not json".to_vec()])+ .collect();+ let excluded = ExcludedPropertyKeys::default();+ let caps = LengthCaps {+ max_property_key_len: 400,+ max_property_value_len: 10_000,+ };++ let mut batch = aggregate_batch(+ &payloads,+ |team_id| team_id == 2,+ &|event: &Event| fan_out(event, &excluded, caps),+ "test",+ );++ let expected = HashMap::from([+ (tuple(2, "$feature/a", "true"), 1_800),+ (tuple(2, "$browser", "Chrome"), 1_800),+ ]);+ assert_eq!(batch.drain(), expected);+ }++ #[test]+ fn drain_ready_counts_empty_messages_toward_the_batch_limit() {+ let mut ready = std::iter::repeat_with(|| Err(RecvErr::Empty))+ .take(2 * MAX_BATCH_MESSAGES)+ .chain((0..MAX_BATCH_MESSAGES).map(|offset| Ok((b"{}".to_vec(), offset))));++ let batch = drain_ready(Ok((b"{}".to_vec(), 0)), || ready.next(), "test");++ assert_eq!(batch.len(), 1);+ assert_eq!(ready.count(), 2 * MAX_BATCH_MESSAGES + 1);+ }++ #[test]+ fn drain_ready_stops_once_payloads_reach_the_byte_budget() {+ let payload = || vec![0; MAX_BATCH_BYTES / 4];+ let mut ready = (1..10).map(|offset| Ok((payload(), offset)));++ let batch = drain_ready(Ok((payload(), 0)), || ready.next(), "test");++ assert_eq!(batch.len(), 4);+ }+#[test]fn cap_top_k_keeps_keys_under_the_cap() {let snap = vec![