:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cohorts): time the tombstone redirect lookup per partition (#127602)
Matheus Batista06a63bcafd02
diff --git a/rust/cohort-stream-processor/src/observability/metrics.rs b/rust/cohort-stream-processor/src/observability/metrics.rsindex 9b543b034c32..587ac3ac8cc7 100644--- a/rust/cohort-stream-processor/src/observability/metrics.rs+++ b/rust/cohort-stream-processor/src/observability/metrics.rs@@ -366,8 +366,13 @@ pub const STAGE1_UNSUPPORTED_VARIANT_SKIPPED: &str = "stage1_unsupported_variant/// The seed path counts once per corrupt row its run reads, not once per tile that touches it;/// `cohort_seed_tiles_dropped_total{reason="corrupt_state"}` is still per tile-leaf.pub const STAGE1_STATE_DECODE_ERROR: &str = "stage1_state_decode_error_total";-/// End-to-end per-event processing latency in the worker (histogram, seconds).+/// Per-event Stage 1 fold latency in the worker (histogram, seconds). It excludes the redirect+/// lookup before the fold ([`STAGE1_REDIRECT_LOOKUP_DURATION`]) and the Stage 2 composition after it.pub const STAGE1_EVENT_PROCESS_DURATION: &str = "stage1_event_process_duration_seconds";+/// Time one event spends resolving its person through the merge tombstones, labelled by `partition`+/// (histogram, seconds). A partition's `rate(..._sum)` is the fraction of its worker's wall time the+/// lookup takes.+pub const STAGE1_REDIRECT_LOOKUP_DURATION: &str = "stage1_redirect_lookup_duration_seconds";/// Keys in the event's single batched Stage-1 pre-read — the reads-per-event distribution/// (histogram).pub const STAGE1_SNAPSHOT_KEYS: &str = "stage1_snapshot_keys";diff --git a/rust/cohort-stream-processor/src/workers/worker.rs b/rust/cohort-stream-processor/src/workers/worker.rsindex af9fd149ccfb..4b25b1efbe3e 100644--- a/rust/cohort-stream-processor/src/workers/worker.rs+++ b/rust/cohort-stream-processor/src/workers/worker.rs@@ -21,7 +21,7 @@ use std::time::{Duration, Instant};use cohort_core::seed::CoverageStartMs;use futures::FutureExt;-use metrics::{counter, gauge, histogram};+use metrics::{counter, gauge, histogram, Histogram};use tokio::task::JoinHandle;use tracing::{error, info, warn};use uuid::Uuid;@@ -40,7 +40,7 @@ use crate::observability::metrics::{MERGE_REKEY_PRODUCE_FAILURE_TOTAL, OUTPUT_MEMBERSHIP_CHANGES_EMITTED, OUTPUT_PRODUCE_ERRORS,OUTPUT_WINDOW_WAITS_TOTAL, SLICES_BEGUN_TOTAL, SLICE_COVERED_SINCE_SECONDS,STAGE1_EVENTS_PROCESSED, STAGE1_EVENTS_SKIPPED, STAGE1_EVENT_PROCESS_DURATION,- STAGE1_STATE_DECODE_ERROR, STAGE1_TRANSITIONS,+ STAGE1_REDIRECT_LOOKUP_DURATION, STAGE1_STATE_DECODE_ERROR, STAGE1_TRANSITIONS,};use crate::partitions::offset_tracker::{MarkOutcome, OffsetTracker};use crate::partitions::router::WorkerInbox;@@ -332,6 +332,7 @@ async fn run_worker(let mut last_yield = Instant::now();let mut last_updated_clock = LastUpdatedClock::default();let mut window = ProduceWindow::new(partition_id, Arc::clone(&sink), Arc::clone(&merge));+ let event_partition = EventPartition::new(partition_id);// Two locals, not `inbox.live`/`inbox.seeds`: both branch futures live at once inside the// macro, so they must borrow disjoint places.let WorkerInbox {@@ -465,7 +466,7 @@ async fn run_worker(Some(max_offset.map_or(cse_offset, |current| current.max(cse_offset)));max_broker_ts = max_broker_ts.max(broker_ts_ms);let effects = handle_event(- partition_id,+ &event_partition,&handle,&catalog,&event,@@ -789,10 +790,28 @@ struct EventEffects {re_keys: Vec<CohortStreamEvent>,}+/// The partition a worker folds events on, with the meters labelled by it. The worker builds it+/// once, so each event records into an already-registered handle instead of a per-event registry+/// lookup, and [`Self::new`] derives the `partition` label from the same id the events fold under.+struct EventPartition {+ id: u16,+ redirect_lookup: Histogram,+}++impl EventPartition {+ fn new(id: u16) -> Self {+ let label: Arc<str> = Arc::from(id.to_string());+ Self {+ id,+ redirect_lookup: histogram!(STAGE1_REDIRECT_LOOKUP_DURATION, "partition" => label),+ }+ }+}+#[allow(clippy::too_many_arguments)]#[hotpath::measure]async fn handle_event(- partition_id: u16,+ partition: &EventPartition,handle: &StoreHandle,catalog: &CatalogHandle,event: &CohortStreamEvent,@@ -800,6 +819,7 @@ async fn handle_event(partition_count: u32,event_name_gating: EventNameGating,) -> EventEffects {+ let partition_id = partition.id;let snapshot = catalog.load();let Some(team_filters) = snapshot.team(TeamId(event.team_id)) else {counter!(STAGE1_EVENTS_SKIPPED, "reason" => SkipReason::NoTeamFilters.as_str())@@ -808,16 +828,20 @@ async fn handle_event(};let filters: &TeamFilters = team_filters;- let resolved: Cow<'_, CohortStreamEvent> =- match redirect_for_tombstone(partition_id, handle, event, partition_count).await {- Redirected::Process(event) => event,- Redirected::ReKey(re_keyed) => {- return EventEffects {- re_keys: vec![re_keyed],- ..EventEffects::default()- }+ let lookup_started = Instant::now();+ let redirected = redirect_for_tombstone(partition_id, handle, event, partition_count).await;+ partition+ .redirect_lookup+ .record(lookup_started.elapsed().as_secs_f64());+ let resolved: Cow<'_, CohortStreamEvent> = match redirected {+ Redirected::Process(event) => event,+ Redirected::ReKey(re_keyed) => {+ return EventEffects {+ re_keys: vec![re_keyed],+ ..EventEffects::default()}- };+ }+ };let started = Instant::now();let result =@@ -1883,7 +1907,7 @@ mod tombstone_redirect_tests {let straggler = person_event(p_old, "[email protected]", 5, 101);let effects = handle_event(- partition_id,+ &EventPartition::new(partition_id),&test_handle(&store),&catalog,&straggler,@@ -2113,10 +2137,13 @@ mod tombstone_redirect_tests {assert_ne!(source_partition, target_partition);write_tombstone(&store, source_partition, p_old, p_new);let handle = test_handle(&store);+ let recorder = metrics_exporter_prometheus::PrometheusBuilder::new().build_recorder();+ let prometheus = recorder.handle();+ let _local = metrics::set_default_local_recorder(&recorder);let straggler = person_event(p_old, "[email protected]", 5, 9);let mut effects = handle_event(- source_partition,+ &EventPartition::new(source_partition),&handle,&catalog,&straggler,@@ -2131,7 +2158,7 @@ mod tombstone_redirect_tests {let re_keyed = effects.re_keys.pop().unwrap();let effects = handle_event(- target_partition,+ &EventPartition::new(target_partition),&handle,&catalog,&re_keyed,@@ -2161,7 +2188,7 @@ mod tombstone_redirect_tests {);let dup = handle_event(- target_partition,+ &EventPartition::new(target_partition),&handle,&catalog,&re_keyed,@@ -2172,6 +2199,17 @@ mod tombstone_redirect_tests {.await;assert!(dup.changes.is_empty(), "the duplicate folds zero times");assert!(dup.re_keys.is_empty());++ let rendered = prometheus.render();+ for (partition, lookups) in [(source_partition, 1), (target_partition, 2)] {+ let series = format!(+ "{STAGE1_REDIRECT_LOOKUP_DURATION}_count{{partition=\"{partition}\"}} {lookups}"+ );+ assert!(+ rendered.lines().any(|line| line == series),+ "the re-keyed lookup counts too, under its own partition: {series} not in\n{rendered}",+ );+ }}#[tokio::test]@@ -2186,7 +2224,7 @@ mod tombstone_redirect_tests {..person_event(p_old, "[email protected]", 5, 9)};let effects = handle_event(- partition_id,+ &EventPartition::new(partition_id),&test_handle(&store),&catalog,&straggler,@@ -2221,7 +2259,7 @@ mod tombstone_redirect_tests {let event = person_event(alice, "[email protected]", 5, 0);let effects = handle_event(- partition_id,+ &EventPartition::new(partition_id),&test_handle(&store),&catalog,&event,