:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): sample the persons the parity monitor compares (#122295)
Zach Brown44a7de0d73cd
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 16c68553f8b5..990c2ddd060d 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -8974,6 +8974,7 @@ dependencies = ["personhog-proto","pin-project","rstest",+ "serde","serde_json","sqlx","thiserror 2.0.18",diff --git a/rust/personhog-common/Cargo.toml b/rust/personhog-common/Cargo.tomlindex 66b7f6d90b6a..66911b7666d9 100644--- a/rust/personhog-common/Cargo.toml+++ b/rust/personhog-common/Cargo.toml@@ -15,6 +15,7 @@ http = { workspace = true }http-body = { workspace = true }http-body-util = { workspace = true }metrics = { workspace = true }+serde = { workspace = true }serde_json = { workspace = true }pin-project = "1.1"sqlx = { workspace = true }diff --git a/rust/personhog-common/src/cdc_parity/person.rs b/rust/personhog-common/src/cdc_parity/person.rsindex 9191b88d0f87..6223d9601c61 100644--- a/rust/personhog-common/src/cdc_parity/person.rs+++ b/rust/personhog-common/src/cdc_parity/person.rs@@ -2,10 +2,12 @@//! the changelog relay produce: folded to the newest version per person, as//! a ReplacingMergeTree keeps it, and compared by content.+use std::borrow::Cow;use std::collections::hash_map::DefaultHasher;use std::collections::{BTreeMap, HashMap, HashSet};use std::hash::Hasher;+use serde::Deserialize;use serde_json::{Map, Value};use uuid::Uuid;@@ -70,11 +72,33 @@ impl PersonRow {})}+ /// Reads the key alone. The properties document stays an unparsed+ /// string, so a row outside a sample costs a fraction of `decode`.+ pub fn decode_key(payload: &[u8]) -> Result<PersonKey, DecodeError> {+ let fields: KeyFields<'_> =+ serde_json::from_slice(payload).map_err(|source| DecodeError::Json {+ what: PERSON_ROW,+ source,+ })?;+ let uuid = Uuid::parse_str(&fields.id).map_err(|_| DecodeError::MissingField {+ what: PERSON_ROW,+ field: "a person uuid",+ })?;+ Ok(PersonKey::new(fields.team_id, uuid))+ }+pub fn key(&self) -> PersonKey {PersonKey::new(self.team_id, self.uuid)}}+#[derive(Deserialize)]+struct KeyFields<'a> {+ team_id: i64,+ #[serde(borrow)]+ id: Cow<'a, str>,+}+/// What a feed holds for one person. Properties are kept as a hash per top/// level key, so a held row costs little and a mismatch can still name the/// keys that differ. The version only orders rows; it counts writes, which@@ -268,12 +292,15 @@ mod tests {const UUID: &str = "00000000-0000-0000-0000-000000000001";- fn row(properties: &str, extra: &str) -> PersonRow {- let payload = format!(+ fn payload(properties: &str, extra: &str) -> String {+ format!(r#"{{"id":"{UUID}","created_at":"2023-11-14 22:13:20","properties":{properties},"team_id":5,"is_identified":1,"is_deleted":0,"version":3,"last_seen_at":null{extra}}}"#,properties = serde_json::to_string(properties).unwrap(),- );- PersonRow::decode(payload.as_bytes()).unwrap()+ )+ }++ fn row(properties: &str, extra: &str) -> PersonRow {+ PersonRow::decode(payload(properties, extra).as_bytes()).unwrap()}fn snapshot(properties: &str) -> PersonSnapshot {@@ -284,6 +311,11 @@ mod tests {fn a_node_shaped_row_decodes() {let row = row(r#"{"email":"[email protected]"}"#, "");assert_eq!(row.key(), PersonKey::new(5, Uuid::parse_str(UUID).unwrap()));+ let payload = payload(r#"{"email":"[email protected]"}"#, "");+ assert_eq!(+ PersonRow::decode_key(payload.as_bytes()).unwrap(),+ row.key()+ );assert_eq!(row.version, 3);assert!(row.is_identified && !row.is_deleted);assert_eq!(row.created_at, "2023-11-14 22:13:20");diff --git a/rust/personhog-parity-monitor/README.md b/rust/personhog-parity-monitor/README.mdindex b38e0a074dd8..f2ce22c5ee2a 100644--- a/rust/personhog-parity-monitor/README.md+++ b/rust/personhog-parity-monitor/README.md@@ -33,7 +33,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| `merge_partition_total` | `outcome`: `same`, `different` | Merges present on both feeds, by partition number; only counted while both merge topics have the same partition count || `merge_placement_total` | `feed`, `outcome`: `expected`, `unexpected` | Arrival partition against the key's murmur2 partition || `version_ties_total`, `merge_conflicts_total` | `feed` | A feed contradicting itself |-| `messages_total`, `decode_failures_total`, `read_errors_total`, `metadata_errors_total`, `filtered_total` | `feed`, `topic` | Reader counters |+| `messages_total`, `decode_failures_total`, `read_errors_total`, `metadata_errors_total`, `filtered_total`, `sampled_out_total` | `feed`, `topic` | Reader counters || `topic_partitions` | `feed`, `topic` | Partition counts from metadata, re-read every `PARITY_METADATA_POLL_SECONDS` || `topic_pair_partitions_equal` | `topic` | Whether the two feeds' topics of that kind have the same count || `pending_entries` | `kind` | Entries waiting to settle |@@ -52,6 +52,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| `PERSONHOG_DISTINCT_ID_TOPIC`, `PERSONHOG_MERGE_EVENTS_TOPIC` | required | The publisher's topics || `NODE_PERSON_TOPIC`, `PERSONHOG_PERSON_TOPIC` | unset | Node's and the relay's `clickhouse_person` topics; persons are compared only when both are set || `PARITY_PERSON_IGNORED_PROPERTIES` | `$creator_event_uuid` | Comma separated top level property keys the person comparison leaves out; their values depend on the order each backend applied a merge's writes in |+| `PARITY_PERSON_SAMPLE_PERCENT` | `100` | The share of persons compared, 1 to 100. Both feeds keep the persons whose key hashes into the sample, so every kept person gets a full verdict; a row outside it is read no further than its key and still moves the watermark. Distinct ids and merges are never sampled: a merge's verdict follows other merges and the rows naming its persons || `PARITY_GRACE_SECONDS` | `60` | How long an entry waits for the other feed || `PARITY_IDLE_PARTITION_SECONDS` | `30` | When a quiet partition stops holding the watermark || `PARITY_MAPPING_LAG_MS` | `2000` | How much longer than merges the mappings wait |diff --git a/rust/personhog-parity-monitor/src/config.rs b/rust/personhog-parity-monitor/src/config.rsindex 1121fe29ec7d..6427fc895207 100644--- a/rust/personhog-parity-monitor/src/config.rs+++ b/rust/personhog-parity-monitor/src/config.rs@@ -30,6 +30,11 @@ pub struct Config {/// merge's writes in.#[envconfig(default = "$creator_event_uuid")]pub parity_person_ignored_properties: String,+ /// The share of persons compared, 1 to 100. Both feeds keep the same+ /// persons, picked by a hash of the key, so every kept person still gets a+ /// full verdict.+ #[envconfig(default = "100")]+ pub parity_person_sample_percent: u32,/// Has to exceed the publisher's drain lag, or every pair settles/// one-sided before the personhog copy arrives.@@ -123,6 +128,15 @@ impl Config {.collect()}+ pub fn person_sample_percent(&self) -> Result<u32, String> {+ match self.parity_person_sample_percent {+ percent @ 1..=100 => Ok(percent),+ percent => Err(format!(+ "PARITY_PERSON_SAMPLE_PERCENT is {percent}, expected 1 to 100"+ )),+ }+ }+pub fn teams(&self) -> Result<Option<HashSet<i64>>, String> {let teams = self.parity_teams@@ -199,6 +213,24 @@ mod tests {assert!(config(&[("PARITY_TEAMS", "2,all")]).teams().is_err());}+ #[test]+ fn the_person_sample_accepts_a_percentage_only() {+ for (value, expected) in [+ ("100", Ok(100)),+ ("1", Ok(1)),+ ("0", Err(())),+ ("101", Err(())),+ ] {+ assert_eq!(+ config(&[("PARITY_PERSON_SAMPLE_PERCENT", value)])+ .person_sample_percent()+ .map_err(|_| ()),+ expected,+ "{value}"+ );+ }+ }+#[test]fn persons_are_compared_only_with_both_topics() {let cases = [diff --git a/rust/personhog-parity-monitor/src/feed.rs b/rust/personhog-parity-monitor/src/feed.rsindex f85af46a8b01..36bd26b6ad6f 100644--- a/rust/personhog-parity-monitor/src/feed.rs+++ b/rust/personhog-parity-monitor/src/feed.rs@@ -2,12 +2,14 @@//! nothing committed: a consumer group would split the partitions between//! replicas and break the cross-feed join, and a restart resumes nothing.+use std::collections::hash_map::DefaultHasher;+use std::hash::{Hash, Hasher};use std::sync::Arc;use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use anyhow::{bail, Context, Result};use health::HealthHandle;-use personhog_common::cdc_parity::{DecodeError, DistinctIdRow, MergeEvent, PersonRow};+use personhog_common::cdc_parity::{DecodeError, DistinctIdRow, MergeEvent, PersonKey, PersonRow};use rdkafka::consumer::{Consumer, StreamConsumer};use rdkafka::error::KafkaError;use rdkafka::message::BorrowedMessage;@@ -26,15 +28,43 @@ const ERROR_BACKOFF: Duration = Duration::from_secs(1);/// librdkafka's default prefetch is 64 MiB per consumer.const PREFETCH_KBYTES: &str = "16384";+/// Picks the persons both feeds compare. The pick depends on the key alone,+/// so a kept person is kept on both feeds and never settles one-sided.+#[derive(Debug, Clone, Copy)]+pub struct PersonSample {+ percent: u32,+}++impl PersonSample {+ pub fn new(percent: u32) -> Self {+ Self { percent }+ }++ fn is_partial(self) -> bool {+ self.percent < 100+ }++ pub fn includes(self, key: &PersonKey) -> bool {+ let mut hasher = DefaultHasher::new();+ key.hash(&mut hasher);+ hasher.finish() % 100 < u64::from(self.percent)+ }+}+pub struct TopicReader {topic: Topic,name: String,consumer: Arc<StreamConsumer>,partitions: Vec<i32>,+ person_sample: PersonSample,}impl TopicReader {- pub fn connect(feed: &FeedConfig, kind: Kind) -> Result<(Self, u32)> {+ pub fn connect(+ feed: &FeedConfig,+ kind: Kind,+ person_sample: PersonSample,+ ) -> Result<(Self, u32)> {let name = match kind {Kind::DistinctIds => feed.distinct_id_topic.clone(),Kind::MergeEvents => feed.merge_events_topic.clone(),@@ -89,6 +119,7 @@ impl TopicReader {name,consumer: Arc::new(consumer),partitions,+ person_sample,},count,))@@ -168,7 +199,20 @@ impl TopicReader {};messages.increment(1);let record = match self.decode(&message) {- Ok(record) => record,+ Ok(Some(record)) => record,+ // A row outside the sample still moves the watermark, or a+ // small sample would hold every person verdict back.+ Ok(None) => {+ let skipped = Event::Skipped {+ topic: self.topic,+ partition: message.partition(),+ timestamp_ms: message.timestamp().to_millis().unwrap_or_else(now_ms),+ };+ if events.send(skipped).await.is_err() {+ return;+ }+ continue;+ }Err(error) => {decode_failures.increment(1);if decode_failure_logged {@@ -219,12 +263,26 @@ impl TopicReader {Ok(Some(self.partitions.len() as u32))}- fn decode(&self, message: &BorrowedMessage<'_>) -> Result<Record, DecodeError> {+ /// `None` is a person outside the sample, read no further than its key.+ fn decode(&self, message: &BorrowedMessage<'_>) -> Result<Option<Record>, DecodeError> {let payload = message.payload().unwrap_or_default();match self.topic.kind {- Kind::DistinctIds => DistinctIdRow::decode(payload).map(Record::Mapping),- Kind::MergeEvents => MergeEvent::decode(message.key(), payload).map(Record::Merge),- Kind::Persons => PersonRow::decode(payload).map(Record::Person),+ Kind::DistinctIds => {+ DistinctIdRow::decode(payload).map(|row| Some(Record::Mapping(row)))+ }+ Kind::MergeEvents => {+ MergeEvent::decode(message.key(), payload).map(|event| Some(Record::Merge(event)))+ }+ Kind::Persons => {+ if self.person_sample.is_partial()+ && !self+ .person_sample+ .includes(&PersonRow::decode_key(payload)?)+ {+ return Ok(None);+ }+ PersonRow::decode(payload).map(|row| Some(Record::Person(row)))+ }}}}diff --git a/rust/personhog-parity-monitor/src/join.rs b/rust/personhog-parity-monitor/src/join.rsindex 936e868b875a..b3f83403e5d2 100644--- a/rust/personhog-parity-monitor/src/join.rs+++ b/rust/personhog-parity-monitor/src/join.rs@@ -54,12 +54,20 @@ pub enum Event {timestamp_ms: i64,record: Record,},+ /// A row outside the person sample, which still moves the watermark.+ Skipped {+ topic: Topic,+ partition: i32,+ timestamp_ms: i64,+ },}impl Event {pub fn topic(&self) -> Topic {match self {- Event::Assigned { topic, .. } | Event::Arrival { topic, .. } => *topic,+ Event::Assigned { topic, .. }+ | Event::Arrival { topic, .. }+ | Event::Skipped { topic, .. } => *topic,}}}@@ -83,6 +91,7 @@ pub struct JoinConfig {#[derive(Debug, Default, Clone, PartialEq, Eq)]pub struct ObserveReport {pub filtered: bool,+ pub sampled_out: bool,pub version_tie: Option<Side>,pub merge_conflict: Option<Side>,/// Whether the merge event sits on the partition its key selects.@@ -280,6 +289,15 @@ impl Join {self.progress.entry(topic).or_default().partition_count = Some(partitions);return report;}+ Event::Skipped {+ topic,+ partition,+ timestamp_ms,+ } => {+ self.record_progress(topic, partition, timestamp_ms, now);+ report.sampled_out = true;+ return report;+ }Event::Arrival {topic,partition,@@ -288,17 +306,7 @@ impl Join {} => (topic, partition, timestamp_ms, record),};- let progress = self.progress.entry(topic).or_default();- let partition_count = progress.partition_count;- let seen = progress- .partitions- .entry(partition)- .or_insert(PartitionProgress {- max_timestamp_ms: timestamp_ms,- last_seen: now,- });- seen.max_timestamp_ms = seen.max_timestamp_ms.max(timestamp_ms);- seen.last_seen = now;+ let partition_count = self.record_progress(topic, partition, timestamp_ms, now);let team_id = match &record {Record::Mapping(row) => row.team_id,@@ -368,6 +376,27 @@ impl Join {report}+ /// Returns the topic's partition count, which places merge keys.+ fn record_progress(+ &mut self,+ topic: Topic,+ partition: i32,+ timestamp_ms: i64,+ now: Instant,+ ) -> Option<u32> {+ let progress = self.progress.entry(topic).or_default();+ let seen = progress+ .partitions+ .entry(partition)+ .or_insert(PartitionProgress {+ max_timestamp_ms: timestamp_ms,+ last_seen: now,+ });+ seen.max_timestamp_ms = seen.max_timestamp_ms.max(timestamp_ms);+ seen.last_seen = now;+ progress.partition_count+ }+fn warmup_until(&mut self, watermark: i64) -> i64 {if self.config.warmup.is_zero() {return i64::MIN;@@ -857,6 +886,29 @@ mod tests {assert_eq!(mismatch["differing_properties"], json!(["a", "b"]));}+ #[test]+ fn rows_outside_the_person_sample_still_move_the_watermark() {+ let t0 = Instant::now();+ let mut join = join(usize::MAX, None);+ join.observe(person(Side::Node, 1_000, 1, "{}"), t0);+ join.observe(person(Side::Node, 70_000, 2, "{}"), t0);+ let report = join.observe(+ Event::Skipped {+ topic: topic(Side::Personhog, Kind::Persons),+ partition: 0,+ timestamp_ms: 70_000,+ },+ t0,+ );+ assert!(report.sampled_out);++ let report = join.settle(t0);+ assert_eq!(+ report.person_outcomes,+ BTreeMap::from([(PersonOutcome::NodeOnly, 1)])+ );+ }+// Nothing settles until the slower feed passes the grace; the merge// topics never count towards the watermark.#[test]diff --git a/rust/personhog-parity-monitor/src/main.rs b/rust/personhog-parity-monitor/src/main.rsindex 20ab6dc81660..2bbe772a46e5 100644--- a/rust/personhog-parity-monitor/src/main.rs+++ b/rust/personhog-parity-monitor/src/main.rs@@ -11,7 +11,7 @@ use tracing_subscriber::util::SubscriberInitExt;use tracing_subscriber::{fmt, EnvFilter};use personhog_parity_monitor::config::Config;-use personhog_parity_monitor::feed::TopicReader;+use personhog_parity_monitor::feed::{PersonSample, TopicReader};use personhog_parity_monitor::join::{Event, Join, JoinConfig, Kind, SampleRing};use personhog_parity_monitor::{pipeline, server};@@ -42,6 +42,7 @@ async fn main() -> Result<()> {warmup: config.warmup(),person_ignored_properties: config.person_ignored_properties(),};+ let person_sample = config.person_sample_percent().map_err(anyhow::Error::msg)?;let mut kinds = vec![Kind::DistinctIds, Kind::MergeEvents];if config.compares_persons().map_err(anyhow::Error::msg)? {kinds.push(Kind::Persons);@@ -51,6 +52,7 @@ async fn main() -> Result<()> {max_entries = config.parity_max_entries,teams = ?join_config.teams,persons = kinds.contains(&Kind::Persons),+ person_sample_percent = person_sample,debug_api = config.debug_api_secret().is_some(),"starting personhog-parity-monitor");@@ -79,10 +81,12 @@ async fn main() -> Result<()> {for &kind in &kinds {let feed_for_reader = feed.clone();// rdkafka's metadata fetch and assignment block the thread.- let (reader, partitions) =- tokio::task::spawn_blocking(move || TopicReader::connect(&feed_for_reader, kind))- .await- .context("the reader setup task panicked")??;+ let sample = PersonSample::new(person_sample);+ let (reader, partitions) = tokio::task::spawn_blocking(move || {+ TopicReader::connect(&feed_for_reader, kind, sample)+ })+ .await+ .context("the reader setup task panicked")??;let topic = reader.topic();info!(feed = topic.side.label(),diff --git a/rust/personhog-parity-monitor/src/pipeline.rs b/rust/personhog-parity-monitor/src/pipeline.rsindex c583e4c0ab9c..1d1b26b4a5ed 100644--- a/rust/personhog-parity-monitor/src/pipeline.rs+++ b/rust/personhog-parity-monitor/src/pipeline.rs@@ -71,6 +71,10 @@ fn record_observation(topic: Topic, report: &ObserveReport) {metrics::counter!("personhog_parity_filtered_total", "feed" => feed, "topic" => topic.kind.label()).increment(1);}+ if report.sampled_out {+ metrics::counter!("personhog_parity_sampled_out_total", "feed" => feed, "topic" => topic.kind.label())+ .increment(1);+ }if let Some(side) = report.version_tie {metrics::counter!("personhog_parity_version_ties_total", "feed" => side.label()).increment(1);