Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): tombstone merge sources at the row version in the flip (#125188)

Nick Besteedcc843d788
diff --git a/proto/personhog/leader/v1/leader.proto b/proto/personhog/leader/v1/leader.proto
index 1a8962adac6e..ffab4b27d2ae 100644
--- a/proto/personhog/leader/v1/leader.proto
+++ b/proto/personhog/leader/v1/leader.proto
@@ -77,10 +77,12 @@ service PersonHogLeader {
// $set_once fills.
// Scalars fold as created_at = min(target, sources),
// is_identified = true, and last_seen_at = max(target, sources); the
- // version is max(target, max sealed) + 1 — at or above every sealed
- // source's death document, which derives from the same sealed + 1
- // (equal when the highest sealed version dominates; the streams are
- // per-person, so the relation is a convention, not a collision guard).
+ // version is max(target, max sealed) + 1. Each sealed source's death
+ // document derives from the same sealed + 1, so the fold sits at or
+ // above it, and equals it when the highest sealed version dominates.
+ // A source row that moved after its seal raises its death document
+ // above the fold. The streams are per-person, so the relation is a
+ // convention, not a collision guard.
// Safe under at-least-once delivery: re-applying the same fold on
// unchanged target state changes no content and only bumps the version.
// (The standard re-appliable-merge residual applies — see the request's
diff --git a/rust/personhog-identity/.sqlx/query-3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65.json b/rust/personhog-identity/.sqlx/query-3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65.json
new file mode 100644
index 000000000000..d676a2742ea7
--- /dev/null
+++ b/rust/personhog-identity/.sqlx/query-3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "/* service='personhog-identity', operation='merge_flip_record_tombstone_versions' */ \n UPDATE lifecycle_op_person_tmp lop\n SET sealed = lop.sealed || jsonb_build_object('tombstone_version', u.version)\n FROM unnest($2::bigint[], $3::bigint[]) AS u(person_id, version)\n WHERE lop.op_id = $1 AND lop.person_id = u.person_id\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8Array",
+ "Int8Array"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "3f5f548d51430e0d921f06986cfd4181153e1a7eee16480b6f0215b9fdfcdf65"
+}
diff --git a/rust/personhog-identity/.sqlx/query-a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd.json b/rust/personhog-identity/.sqlx/query-a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd.json
new file mode 100644
index 000000000000..f83e7aeedd83
--- /dev/null
+++ b/rust/personhog-identity/.sqlx/query-a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "/* service='personhog-identity', operation='merge_flip_record_tombstone_versions' */ \n UPDATE lifecycle_op_person lop\n SET sealed = lop.sealed || jsonb_build_object('tombstone_version', u.version)\n FROM unnest($2::bigint[], $3::bigint[]) AS u(person_id, version)\n WHERE lop.op_id = $1 AND lop.person_id = u.person_id\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8Array",
+ "Int8Array"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "a1ac3ead430e1602e7b1e2cd7b8b786d13044df463a3d366db0051e807097ffd"
+}
diff --git a/rust/personhog-identity/src/lifecycle/merge.rs b/rust/personhog-identity/src/lifecycle/merge.rs
index 4e3eecd6f8e4..e55eae076482 100644
--- a/rust/personhog-identity/src/lifecycle/merge.rs
+++ b/rust/personhog-identity/src/lifecycle/merge.rs
@@ -251,6 +251,7 @@ const STEPS_TOTAL: &str = "personhog_lifecycle_merge_steps_total";
const OUTCOMES_TOTAL: &str = "personhog_lifecycle_merge_outcomes_total";
/// Pre-flip aborts by refusal slug.
const ABORTS_TOTAL: &str = "personhog_lifecycle_merge_aborts_total";
+const TOMBSTONES_ABOVE_SEAL_TOTAL: &str = "personhog_lifecycle_merge_tombstones_above_seal_total";
fn record_transition(from: &str, to: &str) {
common_metrics::inc(
@@ -355,6 +356,11 @@ struct SealedSnapshot {
// Defaulted so seals written before the field existed still parse.
#[serde(default)]
last_seen_at: Option<i64>,
+ // The version the flip's source tombstone committed at. Absent until
+ // the flip records it; the committed release forwards it so the death
+ // document lands at or above it.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ tombstone_version: Option<i64>,
}
pub struct MergeDriver {
@@ -1494,6 +1500,7 @@ fn snapshot_from_sealed(person: &Person) -> Result<SealedSnapshot, SagaError> {
is_identified: person.is_identified,
properties,
last_seen_at: person.last_seen_at,
+ tombstone_version: None,
})
}
@@ -1839,7 +1846,7 @@ async fn flip(pools: &IdentityPools, tables: &IdentityTables, op: &OpRow) -> Res
record_moved_mappings(&mut tx, tables, op, &sources, &repointed).await?;
move_cohort_membership(&mut tx, tables, &sources, target).await?;
move_hash_key_overrides(&mut tx, tables, team_id, &sources, target).await?;
- tombstone_sealed_sources(&mut tx, tables, op, team_id).await?;
+ let moved_above_seal = tombstone_sealed_sources(&mut tx, tables, op, team_id).await?;
clear_target_mark(&mut tx, tables, op).await?;
if !advance_step_in_tx(
@@ -1859,6 +1866,22 @@ async fn flip(pools: &IdentityPools, tables: &IdentityTables, op: &OpRow) -> Res
MergeStep::DocumentFolded.as_str(),
MergeStep::Flipped.as_str(),
);
+ // Reported after the commit: a driver that loses the step CAS sees the
+ // winner's tombstone as a moved row, and its transaction rolls back.
+ if !moved_above_seal.is_empty() {
+ tracing::warn!(
+ op_id = %op.op_id,
+ team_id = op.team_id,
+ persons = moved_above_seal.len(),
+ person_sealed_tombstone = ?moved_above_seal,
+ "merge source tombstones landed above sealed + 1; the rows moved after the seal"
+ );
+ common_metrics::inc(
+ TOMBSTONES_ABOVE_SEAL_TOTAL,
+ &[],
+ moved_above_seal.len() as u64,
+ );
+ }
Ok(())
}
@@ -2019,15 +2042,19 @@ async fn move_hash_key_overrides(
Ok(())
}
-/// Scrub and tombstone the sealed source person rows at the death version
-/// the leader derives from the same seal (sealed + 1), so a reused key
-/// revives above it.
+/// Scrub and tombstone the sealed source person rows at sealed + 1, the
+/// death version the leader derives from the same seal, so a reused key
+/// revives above it. The fence stops leader writes only: a writer on the
+/// Postgres path can move a source row past its seal before the flip, so
+/// the tombstone also clears the row's own version. The committed version
+/// joins the seal, and the release carries it to the leader. Returns the
+/// rows that moved, as (person_id, sealed_version, tombstone_version).
async fn tombstone_sealed_sources(
tx: &mut Tx<'_>,
tables: &IdentityTables,
op: &OpRow,
team_id: i32,
-) -> Result<(), SagaError> {
+) -> Result<Vec<(i64, i64, i64)>, SagaError> {
let tombstone_sql = format!(
r#"
UPDATE {person_table} p
@@ -2035,22 +2062,51 @@ async fn tombstone_sealed_sources(
properties = '{{}}'::jsonb,
properties_last_updated_at = '{{}}'::jsonb,
properties_last_operation = '{{}}'::jsonb,
- version = (lop.sealed->>'version')::bigint + 1
+ version = GREATEST(COALESCE(p.version, 0) + 1, (lop.sealed->>'version')::bigint + 1)
FROM {lop_table} lop
WHERE lop.op_id = $1 AND lop.role = $3 AND lop.status = $4
AND p.team_id = $2 AND p.id = lop.person_id
+ RETURNING p.id, p.version, (lop.sealed->>'version')::bigint
"#,
person_table = tables.person,
lop_table = tables.lifecycle_op_person,
);
- sqlx::query(&query_tag!("merge_tombstone_sealed_sources", tombstone_sql))
- .bind(op.op_id)
- .bind(team_id)
- .bind(ROLE_SOURCE)
- .bind(STATUS_SEALED)
- .execute(&mut **tx)
- .await?;
- Ok(())
+ let tombstoned_sources: Vec<(i64, i64, Option<i64>)> =
+ sqlx::query_as(&query_tag!("merge_tombstone_sealed_sources", tombstone_sql))
+ .bind(op.op_id)
+ .bind(team_id)
+ .bind(ROLE_SOURCE)
+ .bind(STATUS_SEALED)
+ .fetch_all(&mut **tx)
+ .await?;
+ let moved_above_seal: Vec<(i64, i64, i64)> = tombstoned_sources
+ .iter()
+ .filter_map(|(id, version, sealed)| {
+ sealed
+ .filter(|sealed| *version > sealed + 1)
+ .map(|sealed| (*id, sealed, *version))
+ })
+ .collect();
+
+ let (tombstoned_ids, tombstone_versions): (Vec<i64>, Vec<i64>) = tombstoned_sources
+ .into_iter()
+ .map(|(id, version, _)| (id, version))
+ .unzip();
+ mirrored_query!(
+ tables.is_validation(),
+ op = "merge_flip_record_tombstone_versions",
+ r#"
+ UPDATE {lifecycle_op_person} lop
+ SET sealed = lop.sealed || jsonb_build_object('tombstone_version', u.version)
+ FROM unnest($2::bigint[], $3::bigint[]) AS u(person_id, version)
+ WHERE lop.op_id = $1 AND lop.person_id = u.person_id
+ "#,
+ op.op_id,
+ &tombstoned_ids,
+ &tombstone_versions
+ => execute(&mut **tx)
+ )?;
+ Ok(moved_above_seal)
}
/// The target's own person row is untouched: the folded document
@@ -2116,7 +2172,7 @@ impl MergeDriver {
outcome: ReleaseOutcome::Committed.into(),
sealed_version: Some(snapshot.version),
created_at: snapshot.created_at,
- tombstone_version: None,
+ tombstone_version: snapshot.tombstone_version,
})
.await
.map(|_| ())
diff --git a/rust/personhog-identity/tests/lifecycle_merge_tests.rs b/rust/personhog-identity/tests/lifecycle_merge_tests.rs
index 629a4b24dc79..f30cac751f46 100644
--- a/rust/personhog-identity/tests/lifecycle_merge_tests.rs
+++ b/rust/personhog-identity/tests/lifecycle_merge_tests.rs
@@ -602,6 +602,145 @@ async fn a_merge_folds_repoints_tombstones_and_records_the_outcome() {
h.ctx.cleanup().await.expect("cleanup");
}
+/// The fence stops leader writes only. A writer on the Postgres path can
+/// move a sealed source past its seal before the flip, and a tombstone at
+/// sealed + 1 would then tie or sit below the live row. A second, quiet
+/// source in the same op keeps its own sealed + 1, so the release carries
+/// one tombstone version per source.
+#[tokio::test]
+async fn a_direct_postgres_write_between_seal_and_flip_cannot_outrank_the_source_tombstone() {
+ let h = MergeHarness::new().await;
+ let target = h.ctx.insert_person_with_distinct_id("raced-target").await;
+ let source = h.ctx.insert_person_with_distinct_id("raced-source").await;
+ let quiet = h.ctx.insert_person_with_distinct_id("raced-quiet").await;
+ h.set_person(target, r#"{"a": "target"}"#, 3, false).await;
+ h.set_person(source, r#"{"a": "source"}"#, 7, false).await;
+ h.set_person(quiet, r#"{"a": "quiet"}"#, 2, false).await;
+
+ let op_id = Uuid::now_v7();
+ let request = merge_request("raced-target", &["raced-source", "raced-quiet"]);
+ h.create(op_id, &request).await;
+ h.step(op_id).await.expect("started -> claimed");
+ let row = h.step(op_id).await.expect("claimed -> sources_sealed");
+ assert_eq!(row.step, "sources_sealed");
+
+ sqlx::query(&format!(
+ "UPDATE {} SET version = version + 3 WHERE team_id = $1 AND id = $2",
+ h.ctx.tables.person
+ ))
+ .bind(h.ctx.team_id as i32)
+ .bind(source)
+ .execute(&h.ctx.pool)
+ .await
+ .expect("direct write");
+
+ h.step(op_id)
+ .await
+ .expect("sources_sealed -> document_folded");
+ h.step(op_id).await.expect("document_folded -> flipped");
+ let row = h.step(op_id).await.expect("flipped -> completed");
+ assert!(row.completed_at.is_some());
+ let outcome: MergeOutcome =
+ serde_json::from_value(row.outcome.expect("terminal outcome")).unwrap();
+
+ let (is_deleted, version, _) = h.person_state(source).await;
+ assert!(is_deleted);
+ assert_eq!(
+ version, 11,
+ "the tombstone must outrank the direct write, not just the seal"
+ );
+ let (quiet_deleted, quiet_version, _) = h.person_state(quiet).await;
+ assert!(quiet_deleted);
+ assert_eq!(
+ quiet_version, 3,
+ "an unmoved source tombstones at sealed + 1"
+ );
+ let docs = h.leader.death_documents();
+ assert_eq!(docs.len(), 2);
+ let death_version = |person_id: i64| {
+ docs.iter()
+ .find(|doc| doc.person_id == person_id)
+ .expect("a death document per source")
+ .version
+ };
+ assert_eq!(
+ death_version(source),
+ 11,
+ "the death document honors the tombstone"
+ );
+ assert_eq!(
+ death_version(quiet),
+ 3,
+ "the quiet source's death document is not raised by its partner's move"
+ );
+ let survivor = outcome
+ .survivor
+ .expect("committed merges carry the survivor");
+ assert_eq!(
+ survivor["version"],
+ json!(8),
+ "the fold reads the seals, not the moved row: max(target 3, sealed 7, sealed 2) + 1"
+ );
+}
+
+/// A flip from a build before `tombstone_version` left the seal without it.
+/// The release then carries no floor and the leader falls back to sealed + 1,
+/// which is where that build tombstoned the row.
+#[tokio::test]
+async fn an_op_flipped_by_a_pre_tombstone_version_build_completes_at_the_seal() {
+ let h = MergeHarness::new().await;
+ let target = h.ctx.insert_person_with_distinct_id("upgrade-target").await;
+ let source = h.ctx.insert_person_with_distinct_id("upgrade-source").await;
+ h.set_person(target, r#"{"a": "target"}"#, 3, false).await;
+ h.set_person(source, r#"{"a": "source"}"#, 7, false).await;
+
+ let op_id = Uuid::now_v7();
+ h.create(op_id, &merge_request("upgrade-target", &["upgrade-source"]))
+ .await;
+ h.step(op_id).await.expect("started -> claimed");
+ h.step(op_id).await.expect("claimed -> sources_sealed");
+ h.step(op_id)
+ .await
+ .expect("sources_sealed -> document_folded");
+ let row = h.step(op_id).await.expect("document_folded -> flipped");
+ assert_eq!(row.step, "flipped");
+ assert!(
+ h.op_person_sealed(op_id, source)
+ .await
+ .expect("sealed snapshot")
+ .get("tombstone_version")
+ .is_some(),
+ "this build records the tombstone version at the flip"
+ );
+
+ sqlx::query(&format!(
+ "UPDATE {} SET sealed = sealed - 'tombstone_version' WHERE op_id = $1",
+ h.ctx.tables.lifecycle_op_person
+ ))
+ .bind(op_id)
+ .execute(&h.ctx.pool)
+ .await
+ .expect("strip like a pre-tombstone-version build");
+
+ let row = h.step(op_id).await.expect("flipped -> completed");
+ assert!(row.completed_at.is_some());
+ let (is_deleted, version, _) = h.person_state(source).await;
+ assert!(is_deleted);
+ assert_eq!(version, 8, "the flip tombstoned at sealed + 1");
+ let docs = h.leader.death_documents();
+ assert_eq!(docs.len(), 1);
+ assert_eq!(docs[0].version, 8, "the leader falls back to sealed + 1");
+ assert_eq!(
+ h.leader.calls().last(),
+ Some(&LeaderCall::ReleaseCommitted {
+ person_id: source,
+ sealed_version: 7,
+ mark_status_at_release: Some("sealed".to_string()),
+ }),
+ "the release carries no tombstone version to forward"
+ );
+}
+
#[tokio::test]
async fn a_merge_keeps_a_real_hash_key_over_the_cookieless_sentinel() {
let h = MergeHarness::new().await;
diff --git a/rust/personhog-leader/src/service.rs b/rust/personhog-leader/src/service.rs
index 962097fb2562..6527e1c10ba2 100644
--- a/rust/personhog-leader/src/service.rs
+++ b/rust/personhog-leader/src/service.rs
@@ -1590,11 +1590,14 @@ impl PersonHogLeader for PersonHogLeaderService {
.max();
// The version is a max-merge over the target's floor and every
- // sealed version, plus one: at or above every source's death
- // document, which derives from the same sealed + 1 (equal when
- // the highest sealed version dominates — harmless, the streams
- // are per-person), and re-applying the fold only bumps it again
- // — convergent under at-least-once delivery.
+ // sealed version, plus one. A source's death document starts
+ // from the same sealed + 1, so the fold sits at or above it and
+ // equals it when the highest sealed version dominates. A source
+ // row that moved after its seal raises its death document above
+ // the fold. The streams are per-person, so the relation is a
+ // convention, not a collision guard. Re-applying the fold only
+ // bumps the version again, so it converges under at-least-once
+ // delivery.
let base_version = self
.emitted_versions
.floor_for(partition, &cache_key, person.version);