:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): compare clickhouse_person rows in the parity monitor (#121455)
Zach Brown4cd2c9db9cf7
diff --git a/rust/ingestion-control-plane/src/ui/index.html b/rust/ingestion-control-plane/src/ui/index.htmlindex 7fded89ad5f5..89862f1dc3a2 100644--- a/rust/ingestion-control-plane/src/ui/index.html+++ b/rust/ingestion-control-plane/src/ui/index.html@@ -1731,6 +1731,7 @@summary: el("div", { class: "row", style: "padding: 0 0 10px" }),mappingsBody: el("div", {}, el("div", { class: "empty" }, "Loading…")),mergesBody: el("div", {}, el("div", { class: "empty" }, "Loading…")),+ personsBody: el("div", {}, el("div", { class: "empty" }, "Loading…")),};const matchesAny = (values, q) =>values.some((v) => v !== null && v !== undefined && String(v).toLowerCase().includes(q));@@ -1749,10 +1750,18 @@renderTable: (rows) => this.renderMerges(rows),emptyText: "No merge disagreements in the sample.",});+ this.personsView = listView({+ placeholder: "Filter by team, person, or outcome…",+ pageSize: 100,+ matches: (s, q) => matchesAny([s.team_id, s.person_uuid, s.outcome, ...(s.differing_properties || [])], q),+ renderTable: (rows) => this.renderPersons(rows),+ emptyText: "No person disagreements in the sample. The monitor compares persons only when both person topics are set.",+ });root.append(this.ui.summary,el("div", { class: "panel" }, el("h2", {}, "Distinct ids"), this.ui.mappingsBody),- el("div", { class: "panel" }, el("h2", {}, "Merges"), this.ui.mergesBody)+ el("div", { class: "panel" }, el("h2", {}, "Merges"), this.ui.mergesBody),+ el("div", { class: "panel" }, el("h2", {}, "Persons"), this.ui.personsBody));this.refresh();this.timers.push(setInterval(() => this.refresh(), 30000));@@ -1769,6 +1778,7 @@if (!this.loaded) {this.ui.mappingsBody.replaceChildren(el("div", { class: "empty" }, "No data."));this.ui.mergesBody.replaceChildren(el("div", { class: "empty" }, "No data."));+ this.ui.personsBody.replaceChildren(el("div", { class: "empty" }, "No data."));}return;}@@ -1783,7 +1793,7 @@const samples = data.samples.slice().reverse().sort((a, b) => at(b) - at(a));const counts = new Map();for (const s of samples) {- const label = s.kind === "merge" ? `merge ${s.outcome}` : s.outcome;+ const label = s.kind === "mapping" ? s.outcome : `${s.kind} ${s.outcome}`;counts.set(label, (counts.get(label) || 0) + 1);}this.ui.summary.replaceChildren(@@ -1797,6 +1807,8 @@this.ui.mappingsBody.replaceChildren(this.mappingsView.body);this.mergesView.setRows(samples.filter((s) => s.kind === "merge"));this.ui.mergesBody.replaceChildren(this.mergesView.body);+ this.personsView.setRows(samples.filter((s) => s.kind === "person"));+ this.ui.personsBody.replaceChildren(this.personsView.body);},when(sample) {@@ -1837,6 +1849,33 @@));},+ renderPersons(rows) {+ return el("div", { class: "scroll-x" }, el("table", {},+ el("thead", {}, el("tr", {},+ el("th", {}, "time"), el("th", {}, "team"), el("th", {}, "person"), el("th", {}, "outcome"),+ el("th", { title: "Top level property keys whose values differ or that one side lacks" }, "differing properties"),+ el("th", {}, "deleted"), el("th", {}, "identified"), el("th", {}, "created at"),+ el("th", { title: "Versions order rows within one feed and are not compared across feeds" }, "versions"))),+ el("tbody", {}, rows.map((s) => {+ const node = s.node, personhog = s.personhog;+ const both = node && personhog;+ const pair = (f) => `${node ? f(node) : "–"} / ${personhog ? f(personhog) : "–"}`;+ const differs = (f) => (both && f(node) !== f(personhog) ? "mono diff nowrap" : "mono nowrap");+ const keys = s.differing_properties || [];+ return el("tr", {},+ el("td", { class: "muted mono nowrap" }, this.when(s)),+ el("td", { class: "mono" }, String(s.team_id)),+ el("td", { class: "mono" }, this.shortId(s.person_uuid)),+ el("td", { class: "nowrap" }, el("span", { class: "pill warn" }, s.outcome)),+ el("td", { class: keys.length ? "mono diff" : "muted" }, keys.length ? keys.join(", ") : "–"),+ el("td", { class: differs((r) => r.is_deleted) }, pair((r) => (r.is_deleted ? "yes" : "no"))),+ el("td", { class: differs((r) => r.is_identified) }, pair((r) => (r.is_identified ? "yes" : "no"))),+ el("td", { class: differs((r) => r.created_at) }, pair((r) => r.created_at)),+ el("td", { class: "muted mono nowrap" }, pair((r) => String(r.version))));+ }))+ ));+ },+renderMerges(rows) {const orDash = (v) => (v === null || v === undefined ? "–" : String(v));return el("div", { class: "scroll-x" }, el("table", {},diff --git a/rust/personhog-common/README.md b/rust/personhog-common/README.mdindex 92718a680d09..cfd21a412701 100644--- a/rust/personhog-common/README.md+++ b/rust/personhog-common/README.md@@ -70,4 +70,5 @@ graph TB`src/cdc_parity/` holds the rules for the two person CDC feeds, the `clickhouse_person_distinct_id` rows and the `person_merge_events` that both the Node Postgres path and `personhog-publisher` produce.It covers strict decoding of the wire shapes, the fold to end state (the highest version per distinct id, one merge target per person, survivor chains), and the comparison between two feeds as the Kafka parity e2e (`nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts`) states it: the same mapping per distinct id, the same survivor for every person both feeds named, and every person accounted for.+`person` holds the same for `clickhouse_person` rows, which Node and the changelog relay both produce: the newest version per person, compared by content, never by version.The test harness folds a published topic through it and the parity monitor compares the two feeds through it, so a rule lives in one place.diff --git a/rust/personhog-common/src/cdc_parity/decode.rs b/rust/personhog-common/src/cdc_parity/decode.rsindex 7f72544b207a..9faff534e86b 100644--- a/rust/personhog-common/src/cdc_parity/decode.rs+++ b/rust/personhog-common/src/cdc_parity/decode.rs@@ -30,6 +30,10 @@ pub enum DecodeError {SchemaVersion { value: Value },#[error("merge event with merged_at_ms {value}, expected a positive integer")]MergedAt { value: Value },+ #[error("person row with {field} {value}, expected 0 or 1")]+ PersonFlag { field: &'static str, value: Value },+ #[error("person row whose properties are not a JSON object")]+ Properties,#[error("merge event {old} -> {new} produced under key {actual:?}, expected {expected:?}")]Key {old: Uuid,@@ -158,7 +162,7 @@ impl MergeEvent {}}-fn parse_uuid(value: &Value) -> Option<Uuid> {+pub(super) fn parse_uuid(value: &Value) -> Option<Uuid> {value.as_str().and_then(|s| Uuid::parse_str(s).ok())}diff --git a/rust/personhog-common/src/cdc_parity/mod.rs b/rust/personhog-common/src/cdc_parity/mod.rsindex 619f1e8d1b44..c8a1d3513bd4 100644--- a/rust/personhog-common/src/cdc_parity/mod.rs+++ b/rust/personhog-common/src/cdc_parity/mod.rs@@ -1,12 +1,14 @@-//! The parity rules for the two person CDC feeds: the+//! The parity rules for the person CDC feeds: the//! `clickhouse_person_distinct_id` rows and `person_merge_events` that the-//! Node Postgres path and `personhog-publisher` both produce. Feeds are-//! compared by end state, not message by message, because the two backends-//! take different paths to the same place.+//! Node Postgres path and `personhog-publisher` both produce, and the+//! `clickhouse_person` rows that Node and the changelog relay both produce.+//! Feeds are compared by end state, not message by message, because the two+//! backends take different paths to the same place.pub mod compare;pub mod decode;pub mod feed;+pub mod person;pub use compare::{accounting_violations, compare_mapping, compare_states, compare_survivor, AccountingViolation,@@ -16,3 +18,4 @@ pub use decode::{DecodeError, DistinctIdRow, MergeEvent, MERGE_EVENT_SCHEMA_VERSpub use feed::{DistinctIdKey, FeedState, Mapping, MappingConflict, MergeConflict, MergeLoop, PersonKey,};+pub use person::{compare_person, PersonFeed, PersonOutcome, PersonRow, PersonSnapshot};diff --git a/rust/personhog-common/src/cdc_parity/person.rs b/rust/personhog-common/src/cdc_parity/person.rsnew file mode 100644index 000000000000..9191b88d0f87--- /dev/null+++ b/rust/personhog-common/src/cdc_parity/person.rs@@ -0,0 +1,460 @@+//! Person rows, the `clickhouse_person` shape both the Node Postgres path and+//! the changelog relay produce: folded to the newest version per person, as+//! a ReplacingMergeTree keeps it, and compared by content.++use std::collections::hash_map::DefaultHasher;+use std::collections::{BTreeMap, HashMap, HashSet};+use std::hash::Hasher;++use serde_json::{Map, Value};+use uuid::Uuid;++use super::decode::{parse_uuid, DecodeError};+use super::feed::PersonKey;++const PERSON_ROW: &str = "person row";++#[derive(Debug, Clone, PartialEq)]+pub struct PersonRow {+ pub team_id: i64,+ pub uuid: Uuid,+ pub version: i64,+ pub is_deleted: bool,+ pub is_identified: bool,+ /// ClickHouse's second precision text, the same on both feeds.+ pub created_at: String,+ pub properties: Map<String, Value>,+}++impl PersonRow {+ /// `properties` is a JSON document inside the JSON row.+ pub fn decode(payload: &[u8]) -> Result<Self, DecodeError> {+ let value: Value = serde_json::from_slice(payload).map_err(|source| DecodeError::Json {+ what: PERSON_ROW,+ source,+ })?;+ let missing = |field: &'static str| DecodeError::MissingField {+ what: PERSON_ROW,+ field,+ };+ let flag = |field: &'static str| match value[field].as_u64() {+ Some(0) => Ok(false),+ Some(1) => Ok(true),+ _ => Err(DecodeError::PersonFlag {+ field,+ value: value[field].clone(),+ }),+ };+ let properties = value["properties"]+ .as_str()+ .ok_or_else(|| missing("a properties string"))?;+ let properties = match serde_json::from_str(properties) {+ Ok(Value::Object(properties)) => properties,+ _ => return Err(DecodeError::Properties),+ };+ Ok(Self {+ team_id: value["team_id"]+ .as_i64()+ .ok_or_else(|| missing("an integer team_id"))?,+ uuid: parse_uuid(&value["id"]).ok_or_else(|| missing("a person uuid"))?,+ version: value["version"]+ .as_i64()+ .ok_or_else(|| missing("an integer version"))?,+ is_deleted: flag("is_deleted")?,+ is_identified: flag("is_identified")?,+ created_at: value["created_at"]+ .as_str()+ .ok_or_else(|| missing("a created_at"))?+ .to_string(),+ properties,+ })+ }++ pub fn key(&self) -> PersonKey {+ PersonKey::new(self.team_id, self.uuid)+ }+}++/// What a feed holds for one person. Properties are kept as a hash per top+/// level key, so a held row costs little and a mismatch can still name the+/// keys that differ. The version only orders rows; it counts writes, which+/// the two backends batch differently, so it is never compared across feeds.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct PersonSnapshot {+ pub version: i64,+ pub is_deleted: bool,+ pub is_identified: bool,+ pub created_at: String,+ pub properties: BTreeMap<String, u64>,+}++impl PersonSnapshot {+ pub fn new(row: &PersonRow, ignored: &HashSet<String>) -> Self {+ Self {+ version: row.version,+ is_deleted: row.is_deleted,+ is_identified: row.is_identified,+ created_at: row.created_at.clone(),+ properties: row+ .properties+ .iter()+ .filter(|(key, _)| !ignored.contains(*key))+ .map(|(key, value)| (key.clone(), value_hash(value)))+ .collect(),+ }+ }++ pub fn differing_keys<'a>(&'a self, other: &'a PersonSnapshot) -> Vec<&'a str> {+ let mut keys: Vec<&str> = self+ .properties+ .keys()+ .chain(other.properties.keys())+ .filter(|key| self.properties.get(*key) != other.properties.get(*key))+ .map(String::as_str)+ .collect();+ keys.sort_unstable();+ keys.dedup();+ keys+ }+}++/// Key order never counts. Numbers count by their f64 value: Node writes+/// properties through JavaScript, which reads `1.0` as `1` and rounds+/// integers past 2^53, so only what both sides can express is compared.+fn value_hash(value: &Value) -> u64 {+ fn write(hasher: &mut DefaultHasher, value: &Value) {+ match value {+ Value::Null => hasher.write_u8(0),+ Value::Bool(b) => {+ hasher.write_u8(1);+ hasher.write_u8(u8::from(*b));+ }+ Value::Number(n) => {+ hasher.write_u8(2);+ let n = n.as_f64().unwrap_or(f64::NAN);+ // -0 and 0 print the same in JavaScript.+ hasher.write_u64(if n == 0.0 { 0 } else { n.to_bits() });+ }+ Value::String(s) => {+ hasher.write_u8(3);+ hasher.write(s.as_bytes());+ hasher.write_u8(0xff);+ }+ Value::Array(items) => {+ hasher.write_u8(4);+ hasher.write_usize(items.len());+ for item in items {+ write(hasher, item);+ }+ }+ Value::Object(map) => {+ hasher.write_u8(5);+ hasher.write_usize(map.len());+ let mut keys: Vec<&String> = map.keys().collect();+ keys.sort_unstable();+ for key in keys {+ hasher.write(key.as_bytes());+ hasher.write_u8(0xff);+ write(hasher, &map[key]);+ }+ }+ }+ }+ let mut hasher = DefaultHasher::new();+ write(&mut hasher, value);+ hasher.finish()+}++/// A deletion difference outranks a content difference; properties outrank+/// the flags.+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub enum PersonOutcome {+ Agree,+ DeletedMismatch,+ PropertiesMismatch,+ IdentifiedMismatch,+ CreatedAtMismatch,+ NodeOnly,+ PersonhogOnly,+}++impl PersonOutcome {+ pub fn label(self) -> &'static str {+ match self {+ PersonOutcome::Agree => "agree",+ PersonOutcome::DeletedMismatch => "deleted_mismatch",+ PersonOutcome::PropertiesMismatch => "properties_mismatch",+ PersonOutcome::IdentifiedMismatch => "identified_mismatch",+ PersonOutcome::CreatedAtMismatch => "created_at_mismatch",+ PersonOutcome::NodeOnly => "node_only",+ PersonOutcome::PersonhogOnly => "personhog_only",+ }+ }+}++/// Two deleted persons agree whatever else their rows carry: each backend+/// writes its own death row, and both empty the properties.+pub fn compare_person(+ node: Option<&PersonSnapshot>,+ personhog: Option<&PersonSnapshot>,+) -> PersonOutcome {+ match (node, personhog) {+ (None, None) => PersonOutcome::Agree,+ (Some(_), None) => PersonOutcome::NodeOnly,+ (None, Some(_)) => PersonOutcome::PersonhogOnly,+ (Some(node), Some(personhog)) => {+ if node.is_deleted != personhog.is_deleted {+ PersonOutcome::DeletedMismatch+ } else if node.is_deleted {+ PersonOutcome::Agree+ } else if node.properties != personhog.properties {+ PersonOutcome::PropertiesMismatch+ } else if node.is_identified != personhog.is_identified {+ PersonOutcome::IdentifiedMismatch+ } else if node.created_at != personhog.created_at {+ PersonOutcome::CreatedAtMismatch+ } else {+ PersonOutcome::Agree+ }+ }+ }+}++#[derive(Debug, Default)]+pub struct PersonFeed {+ latest: HashMap<PersonKey, PersonSnapshot>,+}++impl PersonFeed {+ /// Keeps the newest version, the first arrival among equal ones. True+ /// when a different row arrives at the version held: the feed+ /// contradicting itself.+ pub fn observe(&mut self, row: &PersonRow, ignored: &HashSet<String>) -> bool {+ let snapshot = PersonSnapshot::new(row, ignored);+ match self.latest.get_mut(&row.key()) {+ Some(held) if held.version > snapshot.version => false,+ Some(held) if held.version == snapshot.version => *held != snapshot,+ Some(held) => {+ *held = snapshot;+ false+ }+ None => {+ self.latest.insert(row.key(), snapshot);+ false+ }+ }+ }++ pub fn get(&self, key: &PersonKey) -> Option<&PersonSnapshot> {+ self.latest.get(key)+ }++ pub fn remove(&mut self, key: &PersonKey) {+ self.latest.remove(key);+ }++ pub fn len(&self) -> usize {+ self.latest.len()+ }++ pub fn is_empty(&self) -> bool {+ self.latest.is_empty()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ const UUID: &str = "00000000-0000-0000-0000-000000000001";++ fn row(properties: &str, extra: &str) -> PersonRow {+ let payload = format!(+ r#"{{"id":"{UUID}","created_at":"2023-11-14 22:13:20","properties":{properties},"team_id":5,"is_identified":1,"is_deleted":0,"version":3,"last_seen_at":null{extra}}}"#,+ properties = serde_json::to_string(properties).unwrap(),+ );+ PersonRow::decode(payload.as_bytes()).unwrap()+ }++ fn snapshot(properties: &str) -> PersonSnapshot {+ PersonSnapshot::new(&row(properties, ""), &HashSet::new())+ }++ #[test]+ fn a_node_shaped_row_decodes() {+ let row = row(r#"{"email":"[email protected]"}"#, "");+ assert_eq!(row.key(), PersonKey::new(5, Uuid::parse_str(UUID).unwrap()));+ assert_eq!(row.version, 3);+ assert!(row.is_identified && !row.is_deleted);+ assert_eq!(row.created_at, "2023-11-14 22:13:20");+ assert_eq!(row.properties["email"], "[email protected]");+ }++ #[test]+ fn rows_a_consumer_would_misread_do_not_decode() {+ let cases = [+ (+ "properties not an object",+ r#"{"id":"00000000-0000-0000-0000-000000000001","created_at":"x","properties":"[1]","team_id":5,"is_identified":1,"is_deleted":0,"version":3}"#,+ ),+ (+ "is_deleted out of range",+ r#"{"id":"00000000-0000-0000-0000-000000000001","created_at":"x","properties":"{}","team_id":5,"is_identified":1,"is_deleted":2,"version":3}"#,+ ),+ (+ "no uuid",+ r#"{"id":"nope","created_at":"x","properties":"{}","team_id":5,"is_identified":1,"is_deleted":0,"version":3}"#,+ ),+ ];+ for (name, payload) in cases {+ assert!(PersonRow::decode(payload.as_bytes()).is_err(), "{name}");+ }+ }++ #[test]+ fn properties_compare_by_what_both_backends_can_express() {+ let cases = [+ (+ "key order",+ r#"{"a":1,"b":{"x":1,"y":2}}"#,+ r#"{"b":{"y":2,"x":1},"a":1}"#,+ true,+ ),+ ("1 against 1.0", r#"{"a":1}"#, r#"{"a":1.0}"#, true),+ (+ "past 2^53",+ r#"{"a":12345678901234567890}"#,+ r#"{"a":12345678901234567000}"#,+ true,+ ),+ ("different values", r#"{"a":1}"#, r#"{"a":2}"#, false),+ (+ "a string against a number",+ r#"{"a":"1"}"#,+ r#"{"a":1}"#,+ false,+ ),+ ("a missing key", r#"{"a":1}"#, r#"{"a":1,"b":null}"#, false),+ ];+ for (name, left, right, equal) in cases {+ assert_eq!(snapshot(left) == snapshot(right), equal, "{name}");+ }+ }++ #[test]+ fn person_comparison_decision_table() {+ let live = snapshot(r#"{"a":1}"#);+ let other_props = snapshot(r#"{"a":2}"#);+ let anonymous = PersonSnapshot {+ is_identified: false,+ ..live.clone()+ };+ let later_created = PersonSnapshot {+ created_at: "2023-11-14 22:13:21".to_string(),+ ..live.clone()+ };+ let deleted = PersonSnapshot {+ is_deleted: true,+ properties: BTreeMap::new(),+ ..live.clone()+ };+ let deleted_elsewhere = PersonSnapshot {+ is_identified: false,+ version: 9,+ ..deleted.clone()+ };+ let cases = [+ ("both absent", None, None, PersonOutcome::Agree),+ (+ "same content, other versions",+ Some(&live),+ Some(&PersonSnapshot {+ version: 7,+ ..live.clone()+ }),+ PersonOutcome::Agree,+ ),+ ("node only", Some(&live), None, PersonOutcome::NodeOnly),+ (+ "personhog only",+ None,+ Some(&live),+ PersonOutcome::PersonhogOnly,+ ),+ (+ "one deleted",+ Some(&live),+ Some(&deleted),+ PersonOutcome::DeletedMismatch,+ ),+ (+ "both deleted, other flags",+ Some(&deleted),+ Some(&deleted_elsewhere),+ PersonOutcome::Agree,+ ),+ (+ "properties",+ Some(&live),+ Some(&other_props),+ PersonOutcome::PropertiesMismatch,+ ),+ (+ "is_identified",+ Some(&live),+ Some(&anonymous),+ PersonOutcome::IdentifiedMismatch,+ ),+ (+ "created_at",+ Some(&live),+ Some(&later_created),+ PersonOutcome::CreatedAtMismatch,+ ),+ ];+ for (name, node, personhog, outcome) in cases {+ assert_eq!(compare_person(node, personhog), outcome, "{name}");+ }+ }++ #[test]+ fn ignored_keys_never_count_and_differing_keys_name_the_rest() {+ let ignored = HashSet::from(["$creator_event_uuid".to_string()]);+ let node = PersonSnapshot::new(+ &row(r#"{"$creator_event_uuid":"x","a":1,"b":1}"#, ""),+ &ignored,+ );+ let personhog = PersonSnapshot::new(+ &row(r#"{"$creator_event_uuid":"y","a":1,"c":1}"#, ""),+ &ignored,+ );+ assert_eq!(node.differing_keys(&personhog), vec!["b", "c"]);+ }++ #[test]+ fn a_feed_keeps_the_newest_row_and_flags_a_contradiction_at_one_version() {+ let mut feed = PersonFeed::default();+ let none = HashSet::new();+ let v3 = row(r#"{"a":1}"#, "");+ let v4 = PersonRow {+ version: 4,+ properties: row(r#"{"a":2}"#, "").properties,+ ..v3.clone()+ };+ let v3_other = PersonRow {+ properties: row(r#"{"a":3}"#, "").properties,+ ..v3.clone()+ };+ assert!(!feed.observe(&v3, &none));+ assert!(!feed.observe(&v4, &none));+ assert!(!feed.observe(&v3_other, &none));+ assert_eq!(feed.get(&v3.key()).unwrap().version, 4);+ assert!(feed.observe(+ &PersonRow {+ version: 4,+ ..v3_other+ },+ &none+ ));+ }+}diff --git a/rust/personhog-parity-monitor/README.md b/rust/personhog-parity-monitor/README.mdindex e74ed2f3e3a0..b38e0a074dd8 100644--- a/rust/personhog-parity-monitor/README.md+++ b/rust/personhog-parity-monitor/README.md@@ -3,6 +3,7 @@A read-only service that follows the two person CDC feeds and reports, as Prometheus metrics, whether they end in the same state.The Node Postgres path produces `clickhouse_person_distinct_id` rows and `person_merge_events`; `personhog-publisher` produces the same two topics from the identity service's outbox.When a lane runs in `shadow` store mode both feeds describe the same events, so every distinct id and every merge should settle identically on both.+With `NODE_PERSON_TOPIC` and `PERSONHOG_PERSON_TOPIC` set, it also compares `clickhouse_person` rows: Node's against those `personhog-changelog-relay` writes from the leaders' changelog.The rules are `personhog_common::cdc_parity`, the ones the Kafka parity e2e (`nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts`) and the test harness gate apply: end state, not messages.Per distinct id only the highest-version row counts, and of it only the person uuid and the deleted flag.@@ -16,7 +17,8 @@ Per merged person only the survivor at the end of its merge chain counts.- Entries older than the first watermark plus `PARITY_WARMUP_SECONDS` (the grace by default) are dropped unjudged and counted as `warmup_evictions_total`: the readers start at the end of the log at different moments, so a row's counterpart may have been produced before its reader started. A fixture whose readers start before any traffic sets it to `0`.- Merges settle first and mappings `PARITY_MAPPING_LAG_MS` later, so a settling merge still sees the rows that named its persons and can tell a stub from a missing merge event.- A merge agrees when both feeds name the same direct target; what became of that target is the target's own verdict. Different direct targets are followed to their survivors, because one feed may reach the same person through a stub the other never made.-- Settled entries are evicted from both feeds. `PARITY_MAX_ENTRIES` bounds the rows and merges retained across both feeds, history included, so a key that keeps receiving versions cannot grow past it behind one entry. Above the cap the oldest entries are evicted down to nine tenths of it without a verdict and counted as overflow. A row or merge costs about half a kilobyte.+- Settled entries are evicted from both feeds. `PARITY_MAX_ENTRIES` bounds the rows, merges and persons retained across both feeds, history included, so a key that keeps receiving versions cannot grow past it behind one entry. Above the cap the oldest entries are evicted down to nine tenths of it without a verdict and counted as overflow. A row or merge costs about half a kilobyte.+- Persons settle against the two person topics' own watermarks, so a lagging relay holds back only person verdicts and shows in `person_watermark_age_seconds`. Each feed keeps its newest row per person. The version only picks that row: it counts writes, which the two backends batch differently. Two deleted persons agree whatever else their death rows carry. Otherwise the properties, `is_identified` and `created_at` must match; properties compare by value, ignoring key order and reading numbers as JavaScript does (`1` equals `1.0`, integers past 2^53 round), without the keys in `PARITY_PERSON_IGNORED_PROPERTIES`. Each side holds a hash per top level property key, so a sample names the keys that differ but never their values. `last_seen_at` is not compared.- Distinct id rows carry null keys on both sides, so their partitions are random and only the partition counts are compared. Merge events are keyed `"{team_id}:{old_person_uuid}"`; each one is checked against `murmur2(key) % partitions` of its own topic, and a merge present on both feeds is checked for landing on the same partition number while both merge topics have the same count.## Metrics@@ -27,6 +29,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| --- | --- | --- || `settled_mappings_total` | `outcome`: `agree`, `person_mismatch`, `deleted_mismatch`, `node_only`, `personhog_only` | One per settled distinct id || `settled_merges_total` | `outcome`: `agree`, `survivor_mismatch`, `node_only`, `personhog_only`, `node_stub`, `personhog_stub`, `loop` | One per settled merged person. `*_stub`: the merging feed created the person inside the window and the other feed never named it, its own path to the result. `*_only`: the other feed named the person, or the person predates the window, and no merge for it arrived |+| `settled_persons_total` | `outcome`: `agree`, `deleted_mismatch`, `properties_mismatch`, `identified_mismatch`, `created_at_mismatch`, `node_only`, `personhog_only` | One per settled person, when the person topics are set || `merge_partition_total` | `outcome`: `same`, `different` | Merges present on both feeds, by partition number; only counted while both merge topics have the same partition count || `merge_placement_total` | `feed`, `outcome`: `expected`, `unexpected` | Arrival partition against the key's murmur2 partition || `version_ties_total`, `merge_conflicts_total` | `feed` | A feed contradicting itself |@@ -35,6 +38,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| `topic_pair_partitions_equal` | `topic` | Whether the two feeds' topics of that kind have the same count || `pending_entries` | `kind` | Entries waiting to settle || `watermark_age_seconds` | `feed` | Wall clock minus the feed's watermark; a stalled feed shows here |+| `person_watermark_age_seconds` | `feed` | The same for the person topics, which persons settle against || `overflow_evictions_total` | | Entries dropped by the cap, so the outcome counters are degraded while this moves || `warmup_evictions_total` | | Entries dropped during the first grace after start |@@ -46,11 +50,13 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| `NODE_DISTINCT_ID_TOPIC`, `NODE_MERGE_EVENTS_TOPIC` | `clickhouse_person_distinct_id`, `person_merge_events` | The Node feed's topics || `PERSONHOG_KAFKA_HOSTS`, `PERSONHOG_KAFKA_TLS` | `localhost:9092`, `false` | The personhog feed's cluster || `PERSONHOG_DISTINCT_ID_TOPIC`, `PERSONHOG_MERGE_EVENTS_TOPIC` | required | The publisher's topics |+| `NODE_PERSON_TOPIC`, `PERSONHOG_PERSON_TOPIC` | unset | Node's and the relay's `clickhouse_person` topics; persons are compared only when both are set |+| `PARITY_PERSON_IGNORED_PROPERTIES` | `$creator_event_uuid` | Comma separated top level property keys the person comparison leaves out; their values depend on the order each backend applied a merge's writes in || `PARITY_GRACE_SECONDS` | `60` | How long an entry waits for the other feed || `PARITY_IDLE_PARTITION_SECONDS` | `30` | When a quiet partition stops holding the watermark || `PARITY_MAPPING_LAG_MS` | `2000` | How much longer than merges the mappings wait || `PARITY_WARMUP_SECONDS` | the grace | How long after the first watermark entries are dropped unjudged; `0` for a fixture whose readers start before any traffic |-| `PARITY_MAX_ENTRIES` | `1000000` | The cap on retained rows and merges across both feeds, about half a gigabyte |+| `PARITY_MAX_ENTRIES` | `1000000` | The cap on retained rows, merges and persons across both feeds, about half a gigabyte; a person with many properties holds a hash per key || `PARITY_TEAMS` | empty, every team | Comma separated team ids; set it to the merge events allowlist when that is a subset || `PARITY_SAMPLE_RING_SIZE` | `500` | Disagreements kept for `/debug/mismatches` || `PARITY_SWEEP_INTERVAL_MS` | `1000` | How often entries are settled |diff --git a/rust/personhog-parity-monitor/src/config.rs b/rust/personhog-parity-monitor/src/config.rsindex c8e24646c875..1121fe29ec7d 100644--- a/rust/personhog-parity-monitor/src/config.rs+++ b/rust/personhog-parity-monitor/src/config.rs@@ -21,6 +21,15 @@ pub struct Config {pub personhog_kafka_tls: bool,pub personhog_distinct_id_topic: String,pub personhog_merge_events_topic: String,+ /// The two `clickhouse_person` feeds; persons are compared only when+ /// both are set.+ pub node_person_topic: Option<String>,+ pub personhog_person_topic: Option<String>,+ /// Comma separated top level property keys left out of the person+ /// comparison: their values depend on the order each backend applied a+ /// merge's writes in.+ #[envconfig(default = "$creator_event_uuid")]+ pub parity_person_ignored_properties: String,/// Has to exceed the publisher's drain lag, or every pair settles/// one-sided before the personhog copy arrives.@@ -35,7 +44,7 @@ pub struct Config {/// Defaults to the grace. Zero turns the warmup off, for a fixture/// whose readers start before any traffic.pub parity_warmup_seconds: Option<u64>,- /// Retained rows and merges across both feeds; each costs about half a+ /// Retained rows, merges and persons across both feeds; each costs about half a/// kilobyte.#[envconfig(default = "1000000")]pub parity_max_entries: usize,@@ -68,6 +77,7 @@ pub struct FeedConfig {pub kafka_tls: bool,pub distinct_id_topic: String,pub merge_events_topic: String,+ pub person_topic: Option<String>,}impl Config {@@ -79,6 +89,7 @@ impl Config {kafka_tls: self.node_kafka_tls,distinct_id_topic: self.node_distinct_id_topic.clone(),merge_events_topic: self.node_merge_events_topic.clone(),+ person_topic: self.node_person_topic.clone(),},FeedConfig {side: Side::Personhog,@@ -86,10 +97,32 @@ impl Config {kafka_tls: self.personhog_kafka_tls,distinct_id_topic: self.personhog_distinct_id_topic.clone(),merge_events_topic: self.personhog_merge_events_topic.clone(),+ person_topic: self.personhog_person_topic.clone(),},]}+ pub fn compares_persons(&self) -> Result<bool, String> {+ let set = |topic: &Option<String>| topic.as_deref().is_some_and(|t| !t.is_empty());+ match (+ set(&self.node_person_topic),+ set(&self.personhog_person_topic),+ ) {+ (true, true) => Ok(true),+ (false, false) => Ok(false),+ _ => Err("set NODE_PERSON_TOPIC and PERSONHOG_PERSON_TOPIC together".to_string()),+ }+ }++ pub fn person_ignored_properties(&self) -> HashSet<String> {+ self.parity_person_ignored_properties+ .split(',')+ .map(str::trim)+ .filter(|key| !key.is_empty())+ .map(str::to_string)+ .collect()+ }+pub fn teams(&self) -> Result<Option<HashSet<i64>>, String> {let teams = self.parity_teams@@ -166,6 +199,35 @@ mod tests {assert!(config(&[("PARITY_TEAMS", "2,all")]).teams().is_err());}+ #[test]+ fn persons_are_compared_only_with_both_topics() {+ let cases = [+ ("neither", vec![], Ok(false)),+ (+ "both",+ vec![("NODE_PERSON_TOPIC", "n"), ("PERSONHOG_PERSON_TOPIC", "p")],+ Ok(true),+ ),+ ("only node", vec![("NODE_PERSON_TOPIC", "n")], Err(())),+ (+ "only personhog",+ vec![("PERSONHOG_PERSON_TOPIC", "p")],+ Err(()),+ ),+ ];+ for (name, vars, expected) in cases {+ assert_eq!(+ config(&vars).compares_persons().map_err(|_| ()),+ expected,+ "{name}"+ );+ }+ assert_eq!(+ config(&[]).person_ignored_properties(),+ HashSet::from(["$creator_event_uuid".to_string()])+ );+ }+#[test]fn the_warmup_follows_the_grace_unless_set() {assert_eq!(diff --git a/rust/personhog-parity-monitor/src/feed.rs b/rust/personhog-parity-monitor/src/feed.rsindex 64ade4f9f732..f85af46a8b01 100644--- a/rust/personhog-parity-monitor/src/feed.rs+++ b/rust/personhog-parity-monitor/src/feed.rs@@ -7,7 +7,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use anyhow::{bail, Context, Result};use health::HealthHandle;-use personhog_common::cdc_parity::{DecodeError, DistinctIdRow, MergeEvent};+use personhog_common::cdc_parity::{DecodeError, DistinctIdRow, MergeEvent, PersonRow};use rdkafka::consumer::{Consumer, StreamConsumer};use rdkafka::error::KafkaError;use rdkafka::message::BorrowedMessage;@@ -38,6 +38,10 @@ impl TopicReader {let name = match kind {Kind::DistinctIds => feed.distinct_id_topic.clone(),Kind::MergeEvents => feed.merge_events_topic.clone(),+ Kind::Persons => feed+ .person_topic+ .clone()+ .context("a person reader without a person topic")?,};let topic = Topic {side: feed.side,@@ -220,6 +224,7 @@ impl TopicReader {match self.topic.kind {Kind::DistinctIds => DistinctIdRow::decode(payload).map(Record::Mapping),Kind::MergeEvents => MergeEvent::decode(message.key(), payload).map(Record::Merge),+ Kind::Persons => PersonRow::decode(payload).map(Record::Person),}}}diff --git a/rust/personhog-parity-monitor/src/join.rs b/rust/personhog-parity-monitor/src/join.rsindex a83c8a367457..936e868b875a 100644--- a/rust/personhog-parity-monitor/src/join.rs+++ b/rust/personhog-parity-monitor/src/join.rs@@ -7,8 +7,9 @@ use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};use personhog_common::cdc_parity::{- compare_mapping, compare_survivor, DistinctIdKey, DistinctIdRow, FeedState, Mapping,- MappingOutcome, MergeEvent, PersonKey, Side, SurvivorOutcome,+ compare_mapping, compare_person, compare_survivor, DistinctIdKey, DistinctIdRow, FeedState,+ Mapping, MappingOutcome, MergeEvent, PersonFeed, PersonKey, PersonOutcome, PersonRow,+ PersonSnapshot, Side, SurvivorOutcome,};use personhog_common::partitioning::partition_for_key;use serde_json::{json, Value};@@ -17,6 +18,7 @@ use serde_json::{json, Value};pub enum Kind {DistinctIds,MergeEvents,+ Persons,}impl Kind {@@ -24,6 +26,7 @@ impl Kind {match self {Kind::DistinctIds => "distinct_ids",Kind::MergeEvents => "merge_events",+ Kind::Persons => "persons",}}}@@ -38,6 +41,7 @@ pub struct Topic {pub enum Record {Mapping(DistinctIdRow),Merge(MergeEvent),+ Person(PersonRow),}#[derive(Debug, Clone)]@@ -73,6 +77,7 @@ pub struct JoinConfig {/// unjudged, because a counterpart may predate the readers. Zero/// turns the warmup off.pub warmup: Duration,+ pub person_ignored_properties: HashSet<String>,}#[derive(Debug, Default, Clone, PartialEq, Eq)]@@ -104,14 +109,17 @@ pub struct SweepReport {pub mapping_outcomes: BTreeMap<MappingOutcome, u64>,pub merge_outcomes: BTreeMap<SurvivorOutcome, u64>,pub merge_partitions: BTreeMap<PartitionAgreement, u64>,+ pub person_outcomes: BTreeMap<PersonOutcome, u64>,pub overflow_evictions: u64,pub warmup_evictions: u64,pub pending_mappings: usize,pub pending_merges: usize,+ pub pending_persons: usize,/// The global watermark the sweep settled against.pub watermark_ms: Option<i64>,/// Each feed's own watermark, indexed by [`side_index`].pub feed_watermarks_ms: [Option<i64>; 2],+ pub person_watermarks_ms: [Option<i64>; 2],}pub fn side_index(side: Side) -> usize {@@ -204,6 +212,10 @@ struct MappingEntry {last_update_ms: i64,}+struct PersonEntry {+ last_update_ms: i64,+}+struct MergeEntry {last_update_ms: i64,partitions: [Option<i32>; 2],@@ -219,6 +231,9 @@ pub struct Join {personhog: FeedState,mappings: HashMap<DistinctIdKey, MappingEntry>,merges: HashMap<PersonKey, MergeEntry>,+ node_persons: PersonFeed,+ personhog_persons: PersonFeed,+ persons: HashMap<PersonKey, PersonEntry>,/// How many merge entries point at each person on either feed.target_refs: HashMap<PersonKey, usize>,progress: HashMap<Topic, TopicProgress>,@@ -234,6 +249,9 @@ impl Join {personhog: FeedState::new(),mappings: HashMap::new(),merges: HashMap::new(),+ node_persons: PersonFeed::default(),+ personhog_persons: PersonFeed::default(),+ persons: HashMap::new(),target_refs: HashMap::new(),progress: HashMap::new(),warmup_until_ms: None,@@ -285,6 +303,7 @@ impl Join {let team_id = match &record {Record::Mapping(row) => row.team_id,Record::Merge(event) => event.team_id,+ Record::Person(row) => row.team_id,};if let Some(teams) = &self.config.teams {if !teams.contains(&team_id) {@@ -331,28 +350,45 @@ impl Join {*self.target_refs.entry(target).or_default() += 1;}}+ Record::Person(row) => {+ let ignored = &self.config.person_ignored_properties;+ let feed = match topic.side {+ Side::Node => &mut self.node_persons,+ Side::Personhog => &mut self.personhog_persons,+ };+ if feed.observe(&row, ignored) {+ report.version_tie = Some(topic.side);+ }+ let entry = self.persons.entry(row.key()).or_insert(PersonEntry {+ last_update_ms: timestamp_ms,+ });+ entry.last_update_ms = entry.last_update_ms.max(timestamp_ms);+ }}report}+ fn warmup_until(&mut self, watermark: i64) -> i64 {+ if self.config.warmup.is_zero() {+ return i64::MIN;+ }+ let warmup = self.config.warmup.as_millis() as i64;+ *self.warmup_until_ms.get_or_insert(watermark + warmup)+ }+/// Merges settle first and mappings a little later, so a merge's/// verdict can still ask which persons each feed named. Entries the/// cap evicts are not counted: they never got their grace.pub fn settle(&mut self, now: Instant) -> SweepReport {let mut report = SweepReport {- feed_watermarks_ms: self.feed_watermarks(now),+ feed_watermarks_ms: self.feed_watermarks(Kind::DistinctIds, now),+ person_watermarks_ms: self.feed_watermarks(Kind::Persons, now),..Default::default()};if let [Some(node), Some(personhog)] = report.feed_watermarks_ms {let watermark = node.min(personhog);report.watermark_ms = Some(watermark);- let warmup_until = if self.config.warmup.is_zero() {- i64::MIN- } else {- *self- .warmup_until_ms- .get_or_insert(watermark + self.config.warmup.as_millis() as i64)- };+ let warmup_until = self.warmup_until(watermark);let merge_cutoff = watermark - self.config.grace.as_millis() as i64;let mapping_cutoff = merge_cutoff - self.config.mapping_lag.as_millis() as i64;let due: Vec<(PersonKey, i64)> = self@@ -385,24 +421,41 @@ impl Join {}}}+ if let [Some(node), Some(personhog)] = report.person_watermarks_ms {+ let watermark = node.min(personhog);+ let warmup_until = self.warmup_until(watermark);+ let cutoff = watermark - self.config.grace.as_millis() as i64;+ let due: Vec<(PersonKey, i64)> = self+ .persons+ .iter()+ .filter(|(_, entry)| entry.last_update_ms <= cutoff)+ .map(|(key, entry)| (*key, entry.last_update_ms))+ .collect();+ for (key, last_update_ms) in due {+ if last_update_ms < warmup_until {+ self.evict_person(&key);+ report.warmup_evictions += 1;+ } else {+ self.settle_person(&key, last_update_ms, &mut report);+ }+ }+ }self.enforce_cap(&mut report);+ report.pending_persons = self.persons.len();report.pending_mappings = self.mappings.len();report.pending_merges = self.merges.values().filter(|entry| !entry.settled).count();report}- /// Only the distinct id topic counts: the sparse merge topic would hold- /// the watermark an idle window behind. A quiet feed advances by the- /// wall clock only while the other feed is quiet too; next to a live- /// feed it has stalled, so its watermark holds and nothing settles.- fn feed_watermarks(&self, now: Instant) -> [Option<i64>; 2] {+ /// Mappings and merges settle on the distinct id topics: the sparse+ /// merge topic would hold the watermark an idle window behind. Persons+ /// settle on the person topics. A quiet feed advances by the wall clock+ /// only while the other feed is quiet too; next to a live feed it has+ /// stalled, so its watermark holds and nothing settles.+ fn feed_watermarks(&self, kind: Kind, now: Instant) -> [Option<i64>; 2] {let idle = self.config.idle_partition;- let progress = [Side::Node, Side::Personhog].map(|side| {- self.progress.get(&Topic {- side,- kind: Kind::DistinctIds,- })- });+ let progress =+ [Side::Node, Side::Personhog].map(|side| self.progress.get(&Topic { side, kind }));let active = progress.map(|topic| topic.and_then(|topic| topic.active_min(now, idle)));let both_quiet = active.iter().all(Option::is_none);let mut watermarks = [None, None];@@ -440,6 +493,35 @@ impl Join {self.evict_mapping(key);}+ fn settle_person(&mut self, key: &PersonKey, last_update_ms: i64, report: &mut SweepReport) {+ let (node, personhog) = (self.node_persons.get(key), self.personhog_persons.get(key));+ let outcome = compare_person(node, personhog);+ if outcome != PersonOutcome::Agree {+ let differing_keys = match (node, personhog) {+ (Some(node), Some(personhog)) => node.differing_keys(personhog),+ _ => Vec::new(),+ };+ self.samples.push(json!({+ "kind": "person",+ "team_id": key.team_id,+ "person_uuid": key.person_uuid.to_string(),+ "outcome": outcome.label(),+ "last_update_ms": last_update_ms,+ "differing_properties": differing_keys,+ "node": node.map(person_json),+ "personhog": personhog.map(person_json),+ }));+ }+ *report.person_outcomes.entry(outcome).or_default() += 1;+ self.evict_person(key);+ }++ fn evict_person(&mut self, key: &PersonKey) {+ self.node_persons.remove(key);+ self.personhog_persons.remove(key);+ self.persons.remove(key);+ }+fn settle_merge(&mut self, old: PersonKey, last_update_ms: i64, report: &mut SweepReport) {let outcome = compare_survivor(&self.node, &self.personhog, old);let partitions = self@@ -532,18 +614,23 @@ impl Join {/// pass per tenth rather than one per sweep, by key age; every key holds/// at least one row, so evicting as many keys as the excess suffices.fn enforce_cap(&mut self, report: &mut SweepReport) {- let total = self.node.row_count() + self.personhog.row_count() + self.merges.len();+ let total = self.node.row_count()+ + self.personhog.row_count()+ + self.merges.len()+ + self.node_persons.len()+ + self.personhog_persons.len();if total <= self.config.max_entries {return;}let target = self.config.max_entries - self.config.max_entries / 10;- let keys = self.mappings.len() + self.merges.len();+ let keys = self.mappings.len() + self.merges.len() + self.persons.len();let excess = (total - target).min(keys);let mut ages: Vec<i64> = self.mappings.values().map(|entry| entry.last_update_ms).chain(self.merges.values().map(|entry| entry.last_update_ms))+ .chain(self.persons.values().map(|entry| entry.last_update_ms)).collect();let (_, threshold, _) = ages.select_nth_unstable(excess - 1);let threshold = *threshold;@@ -567,9 +654,30 @@ impl Join {self.evict_mapping(&key);report.overflow_evictions += 1;}+ let old_persons: Vec<PersonKey> = self+ .persons+ .iter()+ .filter(|(_, entry)| entry.last_update_ms <= threshold)+ .map(|(key, _)| *key)+ .collect();+ for key in old_persons {+ self.evict_person(&key);+ report.overflow_evictions += 1;+ }}}+/// Property values stay out of samples; the differing keys name them.+fn person_json(person: &PersonSnapshot) -> Value {+ json!({+ "is_deleted": person.is_deleted,+ "is_identified": person.is_identified,+ "created_at": person.created_at,+ "version": person.version,+ "property_count": person.properties.len(),+ })+}+fn mapping_json(mapping: &Mapping) -> Value {json!({"person_uuid": mapping.person_uuid.to_string(),@@ -601,6 +709,7 @@ mod tests {teams,mapping_lag: MAPPING_LAG,warmup,+ person_ignored_properties: HashSet::new(),},SampleRing::new(10),)@@ -694,6 +803,60 @@ mod tests {}}+ fn person(side: Side, timestamp_ms: i64, person: u128, properties: &str) -> Event {+ let payload = format!(+ r#"{{"id":"{}","created_at":"2023-11-14 22:13:20","properties":{},"team_id":{TEAM},"is_identified":1,"is_deleted":0,"version":1,"last_seen_at":null}}"#,+ uuid(person),+ serde_json::to_string(properties).unwrap(),+ );+ Event::Arrival {+ topic: topic(side, Kind::Persons),+ partition: 0,+ timestamp_ms,+ record: Record::Person(PersonRow::decode(payload.as_bytes()).unwrap()),+ }+ }++ // Persons settle against the person topics alone: a lagging person feed+ // holds back only person verdicts, and the mappings settle regardless.+ #[test]+ fn persons_settle_behind_their_own_slower_feed() {+ let t0 = Instant::now();+ let mut join = join(usize::MAX, None);+ join.observe(person(Side::Node, 1_000, 1, r#"{"a":1}"#), t0);+ join.observe(person(Side::Personhog, 1_000, 1, r#"{"a":1.0}"#), t0);+ join.observe(person(Side::Node, 1_000, 2, r#"{"a":1,"b":1}"#), t0);+ join.observe(person(Side::Personhog, 1_000, 2, r#"{"a":2}"#), t0);+ join.observe(person(Side::Node, 2_000, 3, "{}"), t0);+ join.observe(person(Side::Node, 70_000, 4, "{}"), t0);+ both(&mut join, 0, 1_000, "d", 9, t0);+ both(&mut join, 0, 70_000, "w", 9, t0);++ let report = join.settle(t0);+ assert_eq!(mapping_outcomes(&report), vec![(MappingOutcome::Agree, 1)]);+ assert_eq!(report.person_outcomes, BTreeMap::new());+ assert_eq!(report.pending_persons, 4);++ join.observe(person(Side::Personhog, 70_000, 4, "{}"), t0);+ let report = join.settle(t0);+ assert_eq!(+ report.person_outcomes,+ BTreeMap::from([+ (PersonOutcome::Agree, 1),+ (PersonOutcome::PropertiesMismatch, 1),+ (PersonOutcome::NodeOnly, 1),+ ])+ );+ assert_eq!(report.pending_persons, 1);+ let samples = join.samples.snapshot();+ let mismatch = samples+ .iter()+ .find(|s| s["outcome"] == "properties_mismatch")+ .expect("a properties sample");+ assert_eq!(mismatch["person_uuid"], uuid(2).to_string());+ assert_eq!(mismatch["differing_properties"], json!(["a", "b"]));+ }+// Nothing settles until the slower feed passes the grace; the merge// topics never count towards the watermark.#[test]@@ -855,6 +1018,7 @@ mod tests {teams: None,mapping_lag: MAPPING_LAG,warmup: Duration::ZERO,+ person_ignored_properties: HashSet::new(),},SampleRing::new(10),);diff --git a/rust/personhog-parity-monitor/src/main.rs b/rust/personhog-parity-monitor/src/main.rsindex 0a65479aae5d..20ab6dc81660 100644--- a/rust/personhog-parity-monitor/src/main.rs+++ b/rust/personhog-parity-monitor/src/main.rs@@ -40,11 +40,17 @@ async fn main() -> Result<()> {teams: config.teams().map_err(anyhow::Error::msg)?,mapping_lag: config.mapping_lag(),warmup: config.warmup(),+ person_ignored_properties: config.person_ignored_properties(),};+ let mut kinds = vec![Kind::DistinctIds, Kind::MergeEvents];+ if config.compares_persons().map_err(anyhow::Error::msg)? {+ kinds.push(Kind::Persons);+ }info!(grace_seconds = config.parity_grace_seconds,max_entries = config.parity_max_entries,teams = ?join_config.teams,+ persons = kinds.contains(&Kind::Persons),debug_api = config.debug_api_secret().is_some(),"starting personhog-parity-monitor");@@ -70,7 +76,7 @@ async fn main() -> Result<()> {));for feed in config.feeds() {- for kind in [Kind::DistinctIds, Kind::MergeEvents] {+ for &kind in &kinds {let feed_for_reader = feed.clone();// rdkafka's metadata fetch and assignment block the thread.let (reader, partitions) =diff --git a/rust/personhog-parity-monitor/src/pipeline.rs b/rust/personhog-parity-monitor/src/pipeline.rsindex 50ccff9f0ad3..c583e4c0ab9c 100644--- a/rust/personhog-parity-monitor/src/pipeline.rs+++ b/rust/personhog-parity-monitor/src/pipeline.rs@@ -53,7 +53,7 @@ fn record_partition_gauges(counts: &HashMap<Topic, u32>) {).set(*count as f64);}- for kind in [Kind::DistinctIds, Kind::MergeEvents] {+ for kind in [Kind::DistinctIds, Kind::MergeEvents, Kind::Persons] {let pair = [Side::Node, Side::Personhog].map(|side| counts.get(&Topic { side, kind }));if let [Some(node), Some(personhog)] = pair {metrics::gauge!(@@ -98,6 +98,10 @@ fn record_sweep(report: &SweepReport) {metrics::counter!("personhog_parity_settled_merges_total", "outcome" => outcome.label()).increment(*count);}+ for (outcome, count) in &report.person_outcomes {+ metrics::counter!("personhog_parity_settled_persons_total", "outcome" => outcome.label())+ .increment(*count);+ }for (agreement, count) in &report.merge_partitions {metrics::counter!("personhog_parity_merge_partition_total", "outcome" => agreement.label()).increment(*count);@@ -114,11 +118,17 @@ fn record_sweep(report: &SweepReport) {.set(report.pending_mappings as f64);metrics::gauge!("personhog_parity_pending_entries", "kind" => "merges").set(report.pending_merges as f64);+ metrics::gauge!("personhog_parity_pending_entries", "kind" => "persons")+ .set(report.pending_persons as f64);let now = now_ms();for side in [Side::Node, Side::Personhog] {if let Some(watermark) = report.feed_watermarks_ms[side_index(side)] {metrics::gauge!("personhog_parity_watermark_age_seconds", "feed" => side.label()).set((now - watermark).max(0) as f64 / 1000.0);}+ if let Some(watermark) = report.person_watermarks_ms[side_index(side)] {+ metrics::gauge!("personhog_parity_person_watermark_age_seconds", "feed" => side.label())+ .set((now - watermark).max(0) as f64 / 1000.0);+ }}}