Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): sample the persons the parity monitor compares (#122295)

Zach Brown44a7de0d73cd
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 16c68553f8b5..990c2ddd060d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -8974,6 +8974,7 @@ dependencies = [
"personhog-proto",
"pin-project",
"rstest",
+ "serde",
"serde_json",
"sqlx",
"thiserror 2.0.18",
diff --git a/rust/personhog-common/Cargo.toml b/rust/personhog-common/Cargo.toml
index 66b7f6d90b6a..66911b7666d9 100644
--- a/rust/personhog-common/Cargo.toml
+++ b/rust/personhog-common/Cargo.toml
@@ -15,6 +15,7 @@ http = { workspace = true }
http-body = { workspace = true }
http-body-util = { workspace = true }
metrics = { workspace = true }
+serde = { workspace = true }
serde_json = { workspace = true }
pin-project = "1.1"
sqlx = { workspace = true }
diff --git a/rust/personhog-common/src/cdc_parity/person.rs b/rust/personhog-common/src/cdc_parity/person.rs
index 9191b88d0f87..6223d9601c61 100644
--- a/rust/personhog-common/src/cdc_parity/person.rs
+++ b/rust/personhog-common/src/cdc_parity/person.rs
@@ -2,10 +2,12 @@
//! the changelog relay produce: folded to the newest version per person, as
//! a ReplacingMergeTree keeps it, and compared by content.
+use std::borrow::Cow;
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::Hasher;
+use serde::Deserialize;
use serde_json::{Map, Value};
use uuid::Uuid;
@@ -70,11 +72,33 @@ impl PersonRow {
})
}
+ /// Reads the key alone. The properties document stays an unparsed
+ /// string, so a row outside a sample costs a fraction of `decode`.
+ pub fn decode_key(payload: &[u8]) -> Result<PersonKey, DecodeError> {
+ let fields: KeyFields<'_> =
+ serde_json::from_slice(payload).map_err(|source| DecodeError::Json {
+ what: PERSON_ROW,
+ source,
+ })?;
+ let uuid = Uuid::parse_str(&fields.id).map_err(|_| DecodeError::MissingField {
+ what: PERSON_ROW,
+ field: "a person uuid",
+ })?;
+ Ok(PersonKey::new(fields.team_id, uuid))
+ }
+
pub fn key(&self) -> PersonKey {
PersonKey::new(self.team_id, self.uuid)
}
}
+#[derive(Deserialize)]
+struct KeyFields<'a> {
+ team_id: i64,
+ #[serde(borrow)]
+ id: Cow<'a, str>,
+}
+
/// What a feed holds for one person. Properties are kept as a hash per top
/// level key, so a held row costs little and a mismatch can still name the
/// keys that differ. The version only orders rows; it counts writes, which
@@ -268,12 +292,15 @@ mod tests {
const UUID: &str = "00000000-0000-0000-0000-000000000001";
- fn row(properties: &str, extra: &str) -> PersonRow {
- let payload = format!(
+ fn payload(properties: &str, extra: &str) -> String {
+ format!(
r#"{{"id":"{UUID}","created_at":"2023-11-14 22:13:20","properties":{properties},"team_id":5,"is_identified":1,"is_deleted":0,"version":3,"last_seen_at":null{extra}}}"#,
properties = serde_json::to_string(properties).unwrap(),
- );
- PersonRow::decode(payload.as_bytes()).unwrap()
+ )
+ }
+
+ fn row(properties: &str, extra: &str) -> PersonRow {
+ PersonRow::decode(payload(properties, extra).as_bytes()).unwrap()
}
fn snapshot(properties: &str) -> PersonSnapshot {
@@ -284,6 +311,11 @@ mod tests {
fn a_node_shaped_row_decodes() {
let row = row(r#"{"email":"[email protected]"}"#, "");
assert_eq!(row.key(), PersonKey::new(5, Uuid::parse_str(UUID).unwrap()));
+ let payload = payload(r#"{"email":"[email protected]"}"#, "");
+ assert_eq!(
+ PersonRow::decode_key(payload.as_bytes()).unwrap(),
+ row.key()
+ );
assert_eq!(row.version, 3);
assert!(row.is_identified && !row.is_deleted);
assert_eq!(row.created_at, "2023-11-14 22:13:20");
diff --git a/rust/personhog-parity-monitor/README.md b/rust/personhog-parity-monitor/README.md
index b38e0a074dd8..f2ce22c5ee2a 100644
--- a/rust/personhog-parity-monitor/README.md
+++ b/rust/personhog-parity-monitor/README.md
@@ -33,7 +33,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te
| `merge_partition_total` | `outcome`: `same`, `different` | Merges present on both feeds, by partition number; only counted while both merge topics have the same partition count |
| `merge_placement_total` | `feed`, `outcome`: `expected`, `unexpected` | Arrival partition against the key's murmur2 partition |
| `version_ties_total`, `merge_conflicts_total` | `feed` | A feed contradicting itself |
-| `messages_total`, `decode_failures_total`, `read_errors_total`, `metadata_errors_total`, `filtered_total` | `feed`, `topic` | Reader counters |
+| `messages_total`, `decode_failures_total`, `read_errors_total`, `metadata_errors_total`, `filtered_total`, `sampled_out_total` | `feed`, `topic` | Reader counters |
| `topic_partitions` | `feed`, `topic` | Partition counts from metadata, re-read every `PARITY_METADATA_POLL_SECONDS` |
| `topic_pair_partitions_equal` | `topic` | Whether the two feeds' topics of that kind have the same count |
| `pending_entries` | `kind` | Entries waiting to settle |
@@ -52,6 +52,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te
| `PERSONHOG_DISTINCT_ID_TOPIC`, `PERSONHOG_MERGE_EVENTS_TOPIC` | required | The publisher's topics |
| `NODE_PERSON_TOPIC`, `PERSONHOG_PERSON_TOPIC` | unset | Node's and the relay's `clickhouse_person` topics; persons are compared only when both are set |
| `PARITY_PERSON_IGNORED_PROPERTIES` | `$creator_event_uuid` | Comma separated top level property keys the person comparison leaves out; their values depend on the order each backend applied a merge's writes in |
+| `PARITY_PERSON_SAMPLE_PERCENT` | `100` | The share of persons compared, 1 to 100. Both feeds keep the persons whose key hashes into the sample, so every kept person gets a full verdict; a row outside it is read no further than its key and still moves the watermark. Distinct ids and merges are never sampled: a merge's verdict follows other merges and the rows naming its persons |
| `PARITY_GRACE_SECONDS` | `60` | How long an entry waits for the other feed |
| `PARITY_IDLE_PARTITION_SECONDS` | `30` | When a quiet partition stops holding the watermark |
| `PARITY_MAPPING_LAG_MS` | `2000` | How much longer than merges the mappings wait |
diff --git a/rust/personhog-parity-monitor/src/config.rs b/rust/personhog-parity-monitor/src/config.rs
index 1121fe29ec7d..6427fc895207 100644
--- a/rust/personhog-parity-monitor/src/config.rs
+++ b/rust/personhog-parity-monitor/src/config.rs
@@ -30,6 +30,11 @@ pub struct Config {
/// merge's writes in.
#[envconfig(default = "$creator_event_uuid")]
pub parity_person_ignored_properties: String,
+ /// The share of persons compared, 1 to 100. Both feeds keep the same
+ /// persons, picked by a hash of the key, so every kept person still gets a
+ /// full verdict.
+ #[envconfig(default = "100")]
+ pub parity_person_sample_percent: u32,
/// Has to exceed the publisher's drain lag, or every pair settles
/// one-sided before the personhog copy arrives.
@@ -123,6 +128,15 @@ impl Config {
.collect()
}
+ pub fn person_sample_percent(&self) -> Result<u32, String> {
+ match self.parity_person_sample_percent {
+ percent @ 1..=100 => Ok(percent),
+ percent => Err(format!(
+ "PARITY_PERSON_SAMPLE_PERCENT is {percent}, expected 1 to 100"
+ )),
+ }
+ }
+
pub fn teams(&self) -> Result<Option<HashSet<i64>>, String> {
let teams = self
.parity_teams
@@ -199,6 +213,24 @@ mod tests {
assert!(config(&[("PARITY_TEAMS", "2,all")]).teams().is_err());
}
+ #[test]
+ fn the_person_sample_accepts_a_percentage_only() {
+ for (value, expected) in [
+ ("100", Ok(100)),
+ ("1", Ok(1)),
+ ("0", Err(())),
+ ("101", Err(())),
+ ] {
+ assert_eq!(
+ config(&[("PARITY_PERSON_SAMPLE_PERCENT", value)])
+ .person_sample_percent()
+ .map_err(|_| ()),
+ expected,
+ "{value}"
+ );
+ }
+ }
+
#[test]
fn persons_are_compared_only_with_both_topics() {
let cases = [
diff --git a/rust/personhog-parity-monitor/src/feed.rs b/rust/personhog-parity-monitor/src/feed.rs
index f85af46a8b01..36bd26b6ad6f 100644
--- a/rust/personhog-parity-monitor/src/feed.rs
+++ b/rust/personhog-parity-monitor/src/feed.rs
@@ -2,12 +2,14 @@
//! nothing committed: a consumer group would split the partitions between
//! replicas and break the cross-feed join, and a restart resumes nothing.
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{bail, Context, Result};
use health::HealthHandle;
-use personhog_common::cdc_parity::{DecodeError, DistinctIdRow, MergeEvent, PersonRow};
+use personhog_common::cdc_parity::{DecodeError, DistinctIdRow, MergeEvent, PersonKey, PersonRow};
use rdkafka::consumer::{Consumer, StreamConsumer};
use rdkafka::error::KafkaError;
use rdkafka::message::BorrowedMessage;
@@ -26,15 +28,43 @@ const ERROR_BACKOFF: Duration = Duration::from_secs(1);
/// librdkafka's default prefetch is 64 MiB per consumer.
const PREFETCH_KBYTES: &str = "16384";
+/// Picks the persons both feeds compare. The pick depends on the key alone,
+/// so a kept person is kept on both feeds and never settles one-sided.
+#[derive(Debug, Clone, Copy)]
+pub struct PersonSample {
+ percent: u32,
+}
+
+impl PersonSample {
+ pub fn new(percent: u32) -> Self {
+ Self { percent }
+ }
+
+ fn is_partial(self) -> bool {
+ self.percent < 100
+ }
+
+ pub fn includes(self, key: &PersonKey) -> bool {
+ let mut hasher = DefaultHasher::new();
+ key.hash(&mut hasher);
+ hasher.finish() % 100 < u64::from(self.percent)
+ }
+}
+
pub struct TopicReader {
topic: Topic,
name: String,
consumer: Arc<StreamConsumer>,
partitions: Vec<i32>,
+ person_sample: PersonSample,
}
impl TopicReader {
- pub fn connect(feed: &FeedConfig, kind: Kind) -> Result<(Self, u32)> {
+ pub fn connect(
+ feed: &FeedConfig,
+ kind: Kind,
+ person_sample: PersonSample,
+ ) -> Result<(Self, u32)> {
let name = match kind {
Kind::DistinctIds => feed.distinct_id_topic.clone(),
Kind::MergeEvents => feed.merge_events_topic.clone(),
@@ -89,6 +119,7 @@ impl TopicReader {
name,
consumer: Arc::new(consumer),
partitions,
+ person_sample,
},
count,
))
@@ -168,7 +199,20 @@ impl TopicReader {
};
messages.increment(1);
let record = match self.decode(&message) {
- Ok(record) => record,
+ Ok(Some(record)) => record,
+ // A row outside the sample still moves the watermark, or a
+ // small sample would hold every person verdict back.
+ Ok(None) => {
+ let skipped = Event::Skipped {
+ topic: self.topic,
+ partition: message.partition(),
+ timestamp_ms: message.timestamp().to_millis().unwrap_or_else(now_ms),
+ };
+ if events.send(skipped).await.is_err() {
+ return;
+ }
+ continue;
+ }
Err(error) => {
decode_failures.increment(1);
if decode_failure_logged {
@@ -219,12 +263,26 @@ impl TopicReader {
Ok(Some(self.partitions.len() as u32))
}
- fn decode(&self, message: &BorrowedMessage<'_>) -> Result<Record, DecodeError> {
+ /// `None` is a person outside the sample, read no further than its key.
+ fn decode(&self, message: &BorrowedMessage<'_>) -> Result<Option<Record>, DecodeError> {
let payload = message.payload().unwrap_or_default();
match self.topic.kind {
- Kind::DistinctIds => DistinctIdRow::decode(payload).map(Record::Mapping),
- Kind::MergeEvents => MergeEvent::decode(message.key(), payload).map(Record::Merge),
- Kind::Persons => PersonRow::decode(payload).map(Record::Person),
+ Kind::DistinctIds => {
+ DistinctIdRow::decode(payload).map(|row| Some(Record::Mapping(row)))
+ }
+ Kind::MergeEvents => {
+ MergeEvent::decode(message.key(), payload).map(|event| Some(Record::Merge(event)))
+ }
+ Kind::Persons => {
+ if self.person_sample.is_partial()
+ && !self
+ .person_sample
+ .includes(&PersonRow::decode_key(payload)?)
+ {
+ return Ok(None);
+ }
+ PersonRow::decode(payload).map(|row| Some(Record::Person(row)))
+ }
}
}
}
diff --git a/rust/personhog-parity-monitor/src/join.rs b/rust/personhog-parity-monitor/src/join.rs
index 936e868b875a..b3f83403e5d2 100644
--- a/rust/personhog-parity-monitor/src/join.rs
+++ b/rust/personhog-parity-monitor/src/join.rs
@@ -54,12 +54,20 @@ pub enum Event {
timestamp_ms: i64,
record: Record,
},
+ /// A row outside the person sample, which still moves the watermark.
+ Skipped {
+ topic: Topic,
+ partition: i32,
+ timestamp_ms: i64,
+ },
}
impl Event {
pub fn topic(&self) -> Topic {
match self {
- Event::Assigned { topic, .. } | Event::Arrival { topic, .. } => *topic,
+ Event::Assigned { topic, .. }
+ | Event::Arrival { topic, .. }
+ | Event::Skipped { topic, .. } => *topic,
}
}
}
@@ -83,6 +91,7 @@ pub struct JoinConfig {
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct ObserveReport {
pub filtered: bool,
+ pub sampled_out: bool,
pub version_tie: Option<Side>,
pub merge_conflict: Option<Side>,
/// Whether the merge event sits on the partition its key selects.
@@ -280,6 +289,15 @@ impl Join {
self.progress.entry(topic).or_default().partition_count = Some(partitions);
return report;
}
+ Event::Skipped {
+ topic,
+ partition,
+ timestamp_ms,
+ } => {
+ self.record_progress(topic, partition, timestamp_ms, now);
+ report.sampled_out = true;
+ return report;
+ }
Event::Arrival {
topic,
partition,
@@ -288,17 +306,7 @@ impl Join {
} => (topic, partition, timestamp_ms, record),
};
- let progress = self.progress.entry(topic).or_default();
- let partition_count = progress.partition_count;
- let seen = progress
- .partitions
- .entry(partition)
- .or_insert(PartitionProgress {
- max_timestamp_ms: timestamp_ms,
- last_seen: now,
- });
- seen.max_timestamp_ms = seen.max_timestamp_ms.max(timestamp_ms);
- seen.last_seen = now;
+ let partition_count = self.record_progress(topic, partition, timestamp_ms, now);
let team_id = match &record {
Record::Mapping(row) => row.team_id,
@@ -368,6 +376,27 @@ impl Join {
report
}
+ /// Returns the topic's partition count, which places merge keys.
+ fn record_progress(
+ &mut self,
+ topic: Topic,
+ partition: i32,
+ timestamp_ms: i64,
+ now: Instant,
+ ) -> Option<u32> {
+ let progress = self.progress.entry(topic).or_default();
+ let seen = progress
+ .partitions
+ .entry(partition)
+ .or_insert(PartitionProgress {
+ max_timestamp_ms: timestamp_ms,
+ last_seen: now,
+ });
+ seen.max_timestamp_ms = seen.max_timestamp_ms.max(timestamp_ms);
+ seen.last_seen = now;
+ progress.partition_count
+ }
+
fn warmup_until(&mut self, watermark: i64) -> i64 {
if self.config.warmup.is_zero() {
return i64::MIN;
@@ -857,6 +886,29 @@ mod tests {
assert_eq!(mismatch["differing_properties"], json!(["a", "b"]));
}
+ #[test]
+ fn rows_outside_the_person_sample_still_move_the_watermark() {
+ let t0 = Instant::now();
+ let mut join = join(usize::MAX, None);
+ join.observe(person(Side::Node, 1_000, 1, "{}"), t0);
+ join.observe(person(Side::Node, 70_000, 2, "{}"), t0);
+ let report = join.observe(
+ Event::Skipped {
+ topic: topic(Side::Personhog, Kind::Persons),
+ partition: 0,
+ timestamp_ms: 70_000,
+ },
+ t0,
+ );
+ assert!(report.sampled_out);
+
+ let report = join.settle(t0);
+ assert_eq!(
+ report.person_outcomes,
+ BTreeMap::from([(PersonOutcome::NodeOnly, 1)])
+ );
+ }
+
// Nothing settles until the slower feed passes the grace; the merge
// topics never count towards the watermark.
#[test]
diff --git a/rust/personhog-parity-monitor/src/main.rs b/rust/personhog-parity-monitor/src/main.rs
index 20ab6dc81660..2bbe772a46e5 100644
--- a/rust/personhog-parity-monitor/src/main.rs
+++ b/rust/personhog-parity-monitor/src/main.rs
@@ -11,7 +11,7 @@ use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter};
use personhog_parity_monitor::config::Config;
-use personhog_parity_monitor::feed::TopicReader;
+use personhog_parity_monitor::feed::{PersonSample, TopicReader};
use personhog_parity_monitor::join::{Event, Join, JoinConfig, Kind, SampleRing};
use personhog_parity_monitor::{pipeline, server};
@@ -42,6 +42,7 @@ async fn main() -> Result<()> {
warmup: config.warmup(),
person_ignored_properties: config.person_ignored_properties(),
};
+ let person_sample = config.person_sample_percent().map_err(anyhow::Error::msg)?;
let mut kinds = vec![Kind::DistinctIds, Kind::MergeEvents];
if config.compares_persons().map_err(anyhow::Error::msg)? {
kinds.push(Kind::Persons);
@@ -51,6 +52,7 @@ async fn main() -> Result<()> {
max_entries = config.parity_max_entries,
teams = ?join_config.teams,
persons = kinds.contains(&Kind::Persons),
+ person_sample_percent = person_sample,
debug_api = config.debug_api_secret().is_some(),
"starting personhog-parity-monitor"
);
@@ -79,10 +81,12 @@ async fn main() -> Result<()> {
for &kind in &kinds {
let feed_for_reader = feed.clone();
// rdkafka's metadata fetch and assignment block the thread.
- let (reader, partitions) =
- tokio::task::spawn_blocking(move || TopicReader::connect(&feed_for_reader, kind))
- .await
- .context("the reader setup task panicked")??;
+ let sample = PersonSample::new(person_sample);
+ let (reader, partitions) = tokio::task::spawn_blocking(move || {
+ TopicReader::connect(&feed_for_reader, kind, sample)
+ })
+ .await
+ .context("the reader setup task panicked")??;
let topic = reader.topic();
info!(
feed = topic.side.label(),
diff --git a/rust/personhog-parity-monitor/src/pipeline.rs b/rust/personhog-parity-monitor/src/pipeline.rs
index c583e4c0ab9c..1d1b26b4a5ed 100644
--- a/rust/personhog-parity-monitor/src/pipeline.rs
+++ b/rust/personhog-parity-monitor/src/pipeline.rs
@@ -71,6 +71,10 @@ fn record_observation(topic: Topic, report: &ObserveReport) {
metrics::counter!("personhog_parity_filtered_total", "feed" => feed, "topic" => topic.kind.label())
.increment(1);
}
+ if report.sampled_out {
+ metrics::counter!("personhog_parity_sampled_out_total", "feed" => feed, "topic" => topic.kind.label())
+ .increment(1);
+ }
if let Some(side) = report.version_tie {
metrics::counter!("personhog_parity_version_ties_total", "feed" => side.label())
.increment(1);