:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(personhog): tombstone merge sources at the row version in the flip (#125188)
Nick Besteedcc843d788
diff --git a/proto/personhog/leader/v1/leader.proto b/proto/personhog/leader/v1/leader.protoindex 1a8962adac6e..ffab4b27d2ae 100644--- a/proto/personhog/leader/v1/leader.proto+++ b/proto/personhog/leader/v1/leader.proto@@ -77,10 +77,12 @@ service PersonHogLeader {// $set_once fills.// Scalars fold as created_at = min(target, sources),// is_identified = true, and last_seen_at = max(target, sources); the- // version is max(target, max sealed) + 1 — at or above every sealed- // source's death document, which derives from the same sealed + 1- // (equal when the highest sealed version dominates; the streams are- // per-person, so the relation is a convention, not a collision guard).+ // version is max(target, max sealed) + 1. Each sealed source's death+ // document derives from the same sealed + 1, so the fold sits at or+ // above it, and equals it when the highest sealed version dominates.+ // A source row that moved after its seal raises its death document+ // above the fold. The streams are per-person, so the relation is a+ // convention, not a collision guard.// Safe under at-least-once delivery: re-applying the same fold on// unchanged target state changes no content and only bumps the version.// (The standard re-appliable-merge residual applies — see the request'sdiff --git a/rust/personhog-identity/.sqlx/query-3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65.json b/rust/personhog-identity/.sqlx/query-3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65.jsonnew file mode 100644index 000000000000..d676a2742ea7--- /dev/null+++ b/rust/personhog-identity/.sqlx/query-3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "/* service='personhog-identity', operation='merge_flip_record_tombstone_versions' */ \n UPDATE lifecycle_op_person_tmp lop\n SET sealed = lop.sealed || jsonb_build_object('tombstone_version', u.version)\n FROM unnest($2::bigint[], $3::bigint[]) AS u(person_id, version)\n WHERE lop.op_id = $1 AND lop.person_id = u.person_id\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8Array",+ "Int8Array"+ ]+ },+ "nullable": []+ },+ "hash": "3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65"+}diff --git a/rust/personhog-identity/.sqlx/query-a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd.json b/rust/personhog-identity/.sqlx/query-a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd.jsonnew file mode 100644index 000000000000..f83e7aeedd83--- /dev/null+++ b/rust/personhog-identity/.sqlx/query-a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "/* service='personhog-identity', operation='merge_flip_record_tombstone_versions' */ \n UPDATE lifecycle_op_person lop\n SET sealed = lop.sealed || jsonb_build_object('tombstone_version', u.version)\n FROM unnest($2::bigint[], $3::bigint[]) AS u(person_id, version)\n WHERE lop.op_id = $1 AND lop.person_id = u.person_id\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8Array",+ "Int8Array"+ ]+ },+ "nullable": []+ },+ "hash": "a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd"+}diff --git a/rust/personhog-identity/src/lifecycle/merge.rs b/rust/personhog-identity/src/lifecycle/merge.rsindex 4e3eecd6f8e4..e55eae076482 100644--- a/rust/personhog-identity/src/lifecycle/merge.rs+++ b/rust/personhog-identity/src/lifecycle/merge.rs@@ -251,6 +251,7 @@ const STEPS_TOTAL: &str = "personhog_lifecycle_merge_steps_total";const OUTCOMES_TOTAL: &str = "personhog_lifecycle_merge_outcomes_total";/// Pre-flip aborts by refusal slug.const ABORTS_TOTAL: &str = "personhog_lifecycle_merge_aborts_total";+const TOMBSTONES_ABOVE_SEAL_TOTAL: &str = "personhog_lifecycle_merge_tombstones_above_seal_total";fn record_transition(from: &str, to: &str) {common_metrics::inc(@@ -355,6 +356,11 @@ struct SealedSnapshot {// Defaulted so seals written before the field existed still parse.#[serde(default)]last_seen_at: Option<i64>,+ // The version the flip's source tombstone committed at. Absent until+ // the flip records it; the committed release forwards it so the death+ // document lands at or above it.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ tombstone_version: Option<i64>,}pub struct MergeDriver {@@ -1494,6 +1500,7 @@ fn snapshot_from_sealed(person: &Person) -> Result<SealedSnapshot, SagaError> {is_identified: person.is_identified,properties,last_seen_at: person.last_seen_at,+ tombstone_version: None,})}@@ -1839,7 +1846,7 @@ async fn flip(pools: &IdentityPools, tables: &IdentityTables, op: &OpRow) -> Resrecord_moved_mappings(&mut tx, tables, op, &sources, &repointed).await?;move_cohort_membership(&mut tx, tables, &sources, target).await?;move_hash_key_overrides(&mut tx, tables, team_id, &sources, target).await?;- tombstone_sealed_sources(&mut tx, tables, op, team_id).await?;+ let moved_above_seal = tombstone_sealed_sources(&mut tx, tables, op, team_id).await?;clear_target_mark(&mut tx, tables, op).await?;if !advance_step_in_tx(@@ -1859,6 +1866,22 @@ async fn flip(pools: &IdentityPools, tables: &IdentityTables, op: &OpRow) -> ResMergeStep::DocumentFolded.as_str(),MergeStep::Flipped.as_str(),);+ // Reported after the commit: a driver that loses the step CAS sees the+ // winner's tombstone as a moved row, and its transaction rolls back.+ if !moved_above_seal.is_empty() {+ tracing::warn!(+ op_id = %op.op_id,+ team_id = op.team_id,+ persons = moved_above_seal.len(),+ person_sealed_tombstone = ?moved_above_seal,+ "merge source tombstones landed above sealed + 1; the rows moved after the seal"+ );+ common_metrics::inc(+ TOMBSTONES_ABOVE_SEAL_TOTAL,+ &[],+ moved_above_seal.len() as u64,+ );+ }Ok(())}@@ -2019,15 +2042,19 @@ async fn move_hash_key_overrides(Ok(())}-/// Scrub and tombstone the sealed source person rows at the death version-/// the leader derives from the same seal (sealed + 1), so a reused key-/// revives above it.+/// Scrub and tombstone the sealed source person rows at sealed + 1, the+/// death version the leader derives from the same seal, so a reused key+/// revives above it. The fence stops leader writes only: a writer on the+/// Postgres path can move a source row past its seal before the flip, so+/// the tombstone also clears the row's own version. The committed version+/// joins the seal, and the release carries it to the leader. Returns the+/// rows that moved, as (person_id, sealed_version, tombstone_version).async fn tombstone_sealed_sources(tx: &mut Tx<'_>,tables: &IdentityTables,op: &OpRow,team_id: i32,-) -> Result<(), SagaError> {+) -> Result<Vec<(i64, i64, i64)>, SagaError> {let tombstone_sql = format!(r#"UPDATE {person_table} p@@ -2035,22 +2062,51 @@ async fn tombstone_sealed_sources(properties = '{{}}'::jsonb,properties_last_updated_at = '{{}}'::jsonb,properties_last_operation = '{{}}'::jsonb,- version = (lop.sealed->>'version')::bigint + 1+ version = GREATEST(COALESCE(p.version, 0) + 1, (lop.sealed->>'version')::bigint + 1)FROM {lop_table} lopWHERE lop.op_id = $1 AND lop.role = $3 AND lop.status = $4AND p.team_id = $2 AND p.id = lop.person_id+ RETURNING p.id, p.version, (lop.sealed->>'version')::bigint"#,person_table = tables.person,lop_table = tables.lifecycle_op_person,);- sqlx::query(&query_tag!("merge_tombstone_sealed_sources", tombstone_sql))- .bind(op.op_id)- .bind(team_id)- .bind(ROLE_SOURCE)- .bind(STATUS_SEALED)- .execute(&mut **tx)- .await?;- Ok(())+ let tombstoned_sources: Vec<(i64, i64, Option<i64>)> =+ sqlx::query_as(&query_tag!("merge_tombstone_sealed_sources", tombstone_sql))+ .bind(op.op_id)+ .bind(team_id)+ .bind(ROLE_SOURCE)+ .bind(STATUS_SEALED)+ .fetch_all(&mut **tx)+ .await?;+ let moved_above_seal: Vec<(i64, i64, i64)> = tombstoned_sources+ .iter()+ .filter_map(|(id, version, sealed)| {+ sealed+ .filter(|sealed| *version > sealed + 1)+ .map(|sealed| (*id, sealed, *version))+ })+ .collect();++ let (tombstoned_ids, tombstone_versions): (Vec<i64>, Vec<i64>) = tombstoned_sources+ .into_iter()+ .map(|(id, version, _)| (id, version))+ .unzip();+ mirrored_query!(+ tables.is_validation(),+ op = "merge_flip_record_tombstone_versions",+ r#"+ UPDATE {lifecycle_op_person} lop+ SET sealed = lop.sealed || jsonb_build_object('tombstone_version', u.version)+ FROM unnest($2::bigint[], $3::bigint[]) AS u(person_id, version)+ WHERE lop.op_id = $1 AND lop.person_id = u.person_id+ "#,+ op.op_id,+ &tombstoned_ids,+ &tombstone_versions+ => execute(&mut **tx)+ )?;+ Ok(moved_above_seal)}/// The target's own person row is untouched: the folded document@@ -2116,7 +2172,7 @@ impl MergeDriver {outcome: ReleaseOutcome::Committed.into(),sealed_version: Some(snapshot.version),created_at: snapshot.created_at,- tombstone_version: None,+ tombstone_version: snapshot.tombstone_version,}).await.map(|_| ())diff --git a/rust/personhog-identity/tests/lifecycle_merge_tests.rs b/rust/personhog-identity/tests/lifecycle_merge_tests.rsindex 629a4b24dc79..f30cac751f46 100644--- a/rust/personhog-identity/tests/lifecycle_merge_tests.rs+++ b/rust/personhog-identity/tests/lifecycle_merge_tests.rs@@ -602,6 +602,145 @@ async fn a_merge_folds_repoints_tombstones_and_records_the_outcome() {h.ctx.cleanup().await.expect("cleanup");}+/// The fence stops leader writes only. A writer on the Postgres path can+/// move a sealed source past its seal before the flip, and a tombstone at+/// sealed + 1 would then tie or sit below the live row. A second, quiet+/// source in the same op keeps its own sealed + 1, so the release carries+/// one tombstone version per source.+#[tokio::test]+async fn a_direct_postgres_write_between_seal_and_flip_cannot_outrank_the_source_tombstone() {+ let h = MergeHarness::new().await;+ let target = h.ctx.insert_person_with_distinct_id("raced-target").await;+ let source = h.ctx.insert_person_with_distinct_id("raced-source").await;+ let quiet = h.ctx.insert_person_with_distinct_id("raced-quiet").await;+ h.set_person(target, r#"{"a": "target"}"#, 3, false).await;+ h.set_person(source, r#"{"a": "source"}"#, 7, false).await;+ h.set_person(quiet, r#"{"a": "quiet"}"#, 2, false).await;++ let op_id = Uuid::now_v7();+ let request = merge_request("raced-target", &["raced-source", "raced-quiet"]);+ h.create(op_id, &request).await;+ h.step(op_id).await.expect("started -> claimed");+ let row = h.step(op_id).await.expect("claimed -> sources_sealed");+ assert_eq!(row.step, "sources_sealed");++ sqlx::query(&format!(+ "UPDATE {} SET version = version + 3 WHERE team_id = $1 AND id = $2",+ h.ctx.tables.person+ ))+ .bind(h.ctx.team_id as i32)+ .bind(source)+ .execute(&h.ctx.pool)+ .await+ .expect("direct write");++ h.step(op_id)+ .await+ .expect("sources_sealed -> document_folded");+ h.step(op_id).await.expect("document_folded -> flipped");+ let row = h.step(op_id).await.expect("flipped -> completed");+ assert!(row.completed_at.is_some());+ let outcome: MergeOutcome =+ serde_json::from_value(row.outcome.expect("terminal outcome")).unwrap();++ let (is_deleted, version, _) = h.person_state(source).await;+ assert!(is_deleted);+ assert_eq!(+ version, 11,+ "the tombstone must outrank the direct write, not just the seal"+ );+ let (quiet_deleted, quiet_version, _) = h.person_state(quiet).await;+ assert!(quiet_deleted);+ assert_eq!(+ quiet_version, 3,+ "an unmoved source tombstones at sealed + 1"+ );+ let docs = h.leader.death_documents();+ assert_eq!(docs.len(), 2);+ let death_version = |person_id: i64| {+ docs.iter()+ .find(|doc| doc.person_id == person_id)+ .expect("a death document per source")+ .version+ };+ assert_eq!(+ death_version(source),+ 11,+ "the death document honors the tombstone"+ );+ assert_eq!(+ death_version(quiet),+ 3,+ "the quiet source's death document is not raised by its partner's move"+ );+ let survivor = outcome+ .survivor+ .expect("committed merges carry the survivor");+ assert_eq!(+ survivor["version"],+ json!(8),+ "the fold reads the seals, not the moved row: max(target 3, sealed 7, sealed 2) + 1"+ );+}++/// A flip from a build before `tombstone_version` left the seal without it.+/// The release then carries no floor and the leader falls back to sealed + 1,+/// which is where that build tombstoned the row.+#[tokio::test]+async fn an_op_flipped_by_a_pre_tombstone_version_build_completes_at_the_seal() {+ let h = MergeHarness::new().await;+ let target = h.ctx.insert_person_with_distinct_id("upgrade-target").await;+ let source = h.ctx.insert_person_with_distinct_id("upgrade-source").await;+ h.set_person(target, r#"{"a": "target"}"#, 3, false).await;+ h.set_person(source, r#"{"a": "source"}"#, 7, false).await;++ let op_id = Uuid::now_v7();+ h.create(op_id, &merge_request("upgrade-target", &["upgrade-source"]))+ .await;+ h.step(op_id).await.expect("started -> claimed");+ h.step(op_id).await.expect("claimed -> sources_sealed");+ h.step(op_id)+ .await+ .expect("sources_sealed -> document_folded");+ let row = h.step(op_id).await.expect("document_folded -> flipped");+ assert_eq!(row.step, "flipped");+ assert!(+ h.op_person_sealed(op_id, source)+ .await+ .expect("sealed snapshot")+ .get("tombstone_version")+ .is_some(),+ "this build records the tombstone version at the flip"+ );++ sqlx::query(&format!(+ "UPDATE {} SET sealed = sealed - 'tombstone_version' WHERE op_id = $1",+ h.ctx.tables.lifecycle_op_person+ ))+ .bind(op_id)+ .execute(&h.ctx.pool)+ .await+ .expect("strip like a pre-tombstone-version build");++ let row = h.step(op_id).await.expect("flipped -> completed");+ assert!(row.completed_at.is_some());+ let (is_deleted, version, _) = h.person_state(source).await;+ assert!(is_deleted);+ assert_eq!(version, 8, "the flip tombstoned at sealed + 1");+ let docs = h.leader.death_documents();+ assert_eq!(docs.len(), 1);+ assert_eq!(docs[0].version, 8, "the leader falls back to sealed + 1");+ assert_eq!(+ h.leader.calls().last(),+ Some(&LeaderCall::ReleaseCommitted {+ person_id: source,+ sealed_version: 7,+ mark_status_at_release: Some("sealed".to_string()),+ }),+ "the release carries no tombstone version to forward"+ );+}+#[tokio::test]async fn a_merge_keeps_a_real_hash_key_over_the_cookieless_sentinel() {let h = MergeHarness::new().await;diff --git a/rust/personhog-leader/src/service.rs b/rust/personhog-leader/src/service.rsindex 962097fb2562..6527e1c10ba2 100644--- a/rust/personhog-leader/src/service.rs+++ b/rust/personhog-leader/src/service.rs@@ -1590,11 +1590,14 @@ impl PersonHogLeader for PersonHogLeaderService {.max();// The version is a max-merge over the target's floor and every- // sealed version, plus one: at or above every source's death- // document, which derives from the same sealed + 1 (equal when- // the highest sealed version dominates — harmless, the streams- // are per-person), and re-applying the fold only bumps it again- // — convergent under at-least-once delivery.+ // sealed version, plus one. A source's death document starts+ // from the same sealed + 1, so the fold sits at or above it and+ // equals it when the highest sealed version dominates. A source+ // row that moved after its seal raises its death document above+ // the fold. The streams are per-person, so the relation is a+ // convention, not a collision guard. Re-applying the fold only+ // bumps the version again, so it converges under at-least-once+ // delivery.let base_version = self.emitted_versions.floor_for(partition, &cache_key, person.version);