:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(personhog): serve a person revived over its cached death document (#128825)
Zach Browna105a0f8e066
diff --git a/rust/personhog-leader/src/service.rs b/rust/personhog-leader/src/service.rsindex 4e89fbe157d2..685d1808174c 100644--- a/rust/personhog-leader/src/service.rs+++ b/rust/personhog-leader/src/service.rs@@ -100,6 +100,22 @@ struct Committed {_inflight: InflightGuard,}+/// Registers the revival outcomes at zero. A revival under a standing+/// death document is rare, so a series born at its first one would hide+/// that one from `increase()`.+fn register_revival_metrics() {+ for outcome in [+ "served",+ "row_below_death",+ "tombstoned",+ "no_fallback",+ "load_error",+ ] {+ counter!("personhog_leader_revivals_total", "outcome" => outcome).increment(0);+ }+ drop(histogram!("personhog_leader_person_load_duration_ms", "source" => "revival"));+}+pub struct PersonHogLeaderService {committer: Arc<Committer>,cache: Arc<PartitionedCache>,@@ -224,6 +240,7 @@ impl PersonHogLeaderService {authority: Arc<AuthorityClock>,emitted_versions: Arc<EmittedVersions>,) -> Self {+ register_revival_metrics();let committer = Arc::new(Committer {cache: Arc::clone(&cache),dirty_index: Arc::clone(&dirty_index),@@ -447,6 +464,73 @@ impl PersonHogLeaderService {}}+ /// A live PG row above the death version can only be a stub-create+ /// revival under the same id. Caller holds the per-key lock.+ async fn serve_revival(+ &self,+ partition: u32,+ key: &PersonCacheKey,+ person: CachedPerson,+ ) -> Result<CachedPerson, Status> {+ if !person.is_deleted {+ return Ok(person);+ }+ let Some(fallback) = &self.fallback else {+ counter!("personhog_leader_revivals_total", "outcome" => "no_fallback").increment(1);+ return Ok(person);+ };+ let started = Instant::now();+ let result = load_person_from_pg(&fallback.pool, &fallback.table, key).await;+ histogram!("personhog_leader_person_load_duration_ms", "source" => "revival")+ .record(started.elapsed().as_secs_f64() * 1000.0);+ match result {+ Ok(Some(row)) if row.version > person.version => {+ counter!("personhog_leader_revivals_total", "outcome" => "served").increment(1);+ tracing::info!(+ team_id = key.team_id,+ person_id = key.person_id,+ death_version = person.version,+ version = row.version,+ "serving a person revived in Postgres over its death document"+ );+ self.cache.put(partition, key.clone(), row.clone());+ Ok(row)+ }+ Ok(Some(row)) => {+ // The tombstone precedes the release, so a live row this+ // low is a pre-death read; it must not revive the person.+ counter!("personhog_leader_revivals_total", "outcome" => "row_below_death")+ .increment(1);+ tracing::warn!(+ team_id = key.team_id,+ person_id = key.person_id,+ death_version = person.version,+ version = row.version,+ "a live Postgres row sits at or below the death document"+ );+ Ok(person)+ }+ Ok(None) => {+ counter!("personhog_leader_revivals_total", "outcome" => "tombstoned").increment(1);+ Ok(person)+ }+ Err(e) => {+ counter!("personhog_leader_revivals_total", "outcome" => "load_error").increment(1);+ counter!("personhog_leader_pg_fallback_errors_total").increment(1);+ tracing::error!(+ team_id = key.team_id,+ person_id = key.person_id,+ error = %e,+ "PG revival check failed"+ );+ // Callers read not-found as gone, which this could not establish.+ Err(Status::unavailable(+ "could not check whether a destroyed person was revived; retry",+ ))+ }+ }+ }+/// Look up a person from cache, falling back to PG on miss./// Acquires a per-key lock.async fn lookup_or_load(@@ -454,12 +538,13 @@ impl PersonHogLeaderService {partition: u32,key: &PersonCacheKey,) -> Result<CachedPerson, Status> {- // Fast path: cache hit (no lock needed)+ // Fast path: cache hit (no lock needed); a death document takes the locked path.match self.cache.get(partition, key) {- CacheLookup::Found(person) => {+ CacheLookup::Found(person) if !person.is_deleted => {Self::record_cache_hit();return Ok(person);}+ CacheLookup::Found(_) => {}CacheLookup::PartitionNotOwned => {return Err(Status::failed_precondition(format!("partition {} not owned by this leader",@@ -474,12 +559,13 @@ impl PersonHogLeaderService {let _guard = mutex.lock().await;// Double-check cache -- another request may have loaded it- if let CacheLookup::Found(person) = self.cache.get(partition, key) {+ let person = if let CacheLookup::Found(person) = self.cache.get(partition, key) {Self::record_cache_hit();- return Ok(person);- }-- self.recover_or_load(partition, key).await+ person+ } else {+ self.recover_or_load(partition, key).await?+ };+ self.serve_revival(partition, key, person).await}/// Look up a person from cache, falling back to PG on miss.@@ -489,17 +575,20 @@ impl PersonHogLeaderService {partition: u32,key: &PersonCacheKey,) -> Result<CachedPerson, Status> {- match self.cache.get(partition, key) {+ let person = match self.cache.get(partition, key) {CacheLookup::Found(person) => {Self::record_cache_hit();- Ok(person)+ person}- CacheLookup::PartitionNotOwned => Err(Status::failed_precondition(format!(- "partition {} not owned by this leader",- partition- ))),- CacheLookup::PersonNotFound => self.recover_or_load(partition, key).await,- }+ CacheLookup::PartitionNotOwned => {+ return Err(Status::failed_precondition(format!(+ "partition {} not owned by this leader",+ partition+ )))+ }+ CacheLookup::PersonNotFound => self.recover_or_load(partition, key).await?,+ };+ self.serve_revival(partition, key, person).await}/// The shared tail of every document write: refuse unapplyable records,diff --git a/rust/personhog-leader/tests/common/mod.rs b/rust/personhog-leader/tests/common/mod.rsindex dd2b9622924b..8d1d867ae2a2 100644--- a/rust/personhog-leader/tests/common/mod.rs+++ b/rust/personhog-leader/tests/common/mod.rs@@ -30,6 +30,9 @@ use personhog_coordination::types::{ChangelogConfig, ChangelogTarget};use personhog_leader::coordination::{Capture, ChangelogCaptures};use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};use rdkafka::client::DefaultClientContext;+use rdkafka::consumer::BaseConsumer;+use rdkafka::error::KafkaError;+use rdkafka::message::BorrowedMessage;use rdkafka::mocking::MockCluster;use rdkafka::producer::{DefaultProducerContext, FutureProducer};use rdkafka::types::RDKafkaErrorCode;@@ -61,6 +64,41 @@ pub const WAIT_TIMEOUT: Duration = Duration::from_secs(10);pub const POLL_INTERVAL: Duration = Duration::from_millis(100);pub const NUM_PARTITIONS: u32 = 4;+/// librdkafka reports a refused broker address and then reconnects on+/// its own, for example when `localhost` resolves to `::1` first. A+/// reader that treats the report as final fails on a broker about to+/// answer, or counts a partition as drained that is not.+pub fn is_transient_broker_error(error: &KafkaError) -> bool {+ matches!(+ error,+ KafkaError::MessageConsumption(+ RDKafkaErrorCode::BrokerTransportFailure | RDKafkaErrorCode::AllBrokersDown+ )+ )+}++/// One record from `consumer`, riding out transient broker errors until+/// `wait` has passed. Panics on any other error and at the deadline.+pub fn poll_record<'a>(+ consumer: &'a BaseConsumer,+ wait: Duration,+ what: &str,+) -> BorrowedMessage<'a> {+ let deadline = std::time::Instant::now() + wait;+ loop {+ match consumer.poll(Duration::from_millis(200)) {+ Some(Ok(message)) => return message,+ Some(Err(error)) if is_transient_broker_error(&error) => {}+ Some(Err(error)) => panic!("kafka error reading {what}: {error}"),+ None => {}+ }+ assert!(+ std::time::Instant::now() < deadline,+ "no message received from {what} within {wait:?}"+ );+ }+}+/// Find a person_id whose key hashes to `partition`, so a test can aim a/// request at a specific partition while satisfying the leader's partition/// validation.diff --git a/rust/personhog-leader/tests/fence_integration.rs b/rust/personhog-leader/tests/fence_integration.rsindex 49475b68e3a4..dfb2087f247d 100644--- a/rust/personhog-leader/tests/fence_integration.rs+++ b/rust/personhog-leader/tests/fence_integration.rs@@ -16,7 +16,7 @@ use common::{use personhog_common::partitioning::partition_for_person;use personhog_common::properties::jsonb_column_size;use personhog_leader::cache::{- CacheLookup, CachedPerson, DirtyIndex, PartitionedCache, PersonCacheKey,+ CacheLookup, CachedPerson, CachedVersion, DirtyIndex, PartitionedCache, PersonCacheKey,};use personhog_leader::emitted::EmittedVersions;use personhog_leader::fence::{@@ -193,11 +193,12 @@ fn changelog_records(harness: &FenceHarness) -> Vec<Person> {let mut records = Vec::new();loop {match consumer.poll(Duration::from_millis(300)) {- Some(result) => {- let msg = result.expect("kafka error");+ Some(Ok(msg)) => {records.push(Person::decode(msg.payload().unwrap()).expect("decode changelog record"));}+ Some(Err(error)) if common::is_transient_broker_error(&error) => {}+ Some(Err(error)) => panic!("kafka error: {error}"),None if !records.is_empty() || std::time::Instant::now() >= deadline => break,None => {}}@@ -488,21 +489,12 @@ async fn a_committed_release_produces_the_death_document_above_every_version() {.expect("cleanup");}-/// While the death document's mark stands, reads answer not-found even if-/// PG already holds a revived row. Once the mark settles, the document-/// leaves the cache and the next read serves the revival from PG.-#[tokio::test]-async fn a_revival_is_served_once_the_death_documents_mark_settles() {- let pool = common::create_persons_pool().await;- let mut harness = start_fence_harness(- test_cached_person(),- Some(PgFallback {- pool: pool.clone(),- table: "posthog_person".to_string(),- lifecycle: LifecycleTables::new("lifecycle_op", "lifecycle_op_person"),- }),- )- .await;+/// Fences and releases the harness person through a delete op marked in+/// PG, leaving its death document cached. Returns the op and the document.+async fn destroy_through_pg(+ harness: &mut FenceHarness,+ pool: &sqlx::PgPool,+) -> (Uuid, CachedVersion) {let team_id = harness.team_id;let partition = harness.partition;let person_id = harness.person_id;@@ -515,7 +507,7 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {).bind(op).bind(team_id as i32)- .execute(&pool)+ .execute(pool).await.expect("insert op");sqlx::query(@@ -525,7 +517,7 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {.bind(op).bind(team_id as i32).bind(person_id)- .execute(&pool)+ .execute(pool).await.expect("insert mark");@@ -564,21 +556,29 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {.peek(partition, &key).expect("the death document is cached");assert!(death.is_deleted);+ (op, death)+}- // A stub-create revived the row in place: same id and uuid, version- // above the death document's.- sqlx::query(- "INSERT INTO posthog_person \- (id, team_id, uuid, properties, created_at, version, is_identified, is_deleted) \- VALUES ($1, $2, $3::uuid, '{\"revived\": true}'::jsonb, now(), $4, false, false)",+/// A live PG row above the death document's version is served over the+/// cached document, over one recovered from the changelog, and past the settle.+#[tokio::test]+async fn a_revival_is_served_over_the_death_document() {+ let pool = common::create_persons_pool().await;+ let mut harness = start_fence_harness(+ test_cached_person(),+ Some(PgFallback {+ pool: pool.clone(),+ table: "posthog_person".to_string(),+ lifecycle: LifecycleTables::new("lifecycle_op", "lifecycle_op_person"),+ }),)- .bind(person_id)- .bind(team_id as i32)- .bind(&person_uuid)- .bind(death.version + 1)- .execute(&pool)- .await- .expect("insert revived row");+ .await;+ let team_id = harness.team_id;+ let partition = harness.partition;+ let person_id = harness.person_id;+ let person_uuid = test_cached_person().uuid;+ let (op, death) = destroy_through_pg(&mut harness, &pool).await;+ let key = PersonCacheKey { team_id, person_id };let get = || {with_partition(@@ -590,14 +590,71 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {partition,)};+ let set_row = |version: i64, is_deleted: bool| {+ let pool = pool.clone();+ let person_uuid = person_uuid.clone();+ async move {+ sqlx::query(+ "INSERT INTO posthog_person \+ (id, team_id, uuid, properties, created_at, version, is_identified, is_deleted) \+ VALUES ($1, $2, $3::uuid, '{\"revived\": true}'::jsonb, now(), $4, false, $5) \+ ON CONFLICT (team_id, id) DO UPDATE \+ SET version = EXCLUDED.version, is_deleted = EXCLUDED.is_deleted",+ )+ .bind(person_id)+ .bind(team_id as i32)+ .bind(&person_uuid)+ .bind(version)+ .bind(is_deleted)+ .execute(&pool)+ .await+ .expect("write row");+ }+ };++ // The saga's own tombstone, as PG holds it between the unmap and the+ // writer's settle.+ set_row(death.version, true).await;+ let status = harness+ .client+ .get_person(get())+ .await+ .expect_err("the death document answers over a tombstoned row");+ assert_eq!(status.code(), Code::NotFound);++ // A live row at the death version is pre-death state, not a revival.+ set_row(death.version, false).await;let status = harness.client.get_person(get()).await- .expect_err("the death document answers while its mark stands");+ .expect_err("the death document still answers");assert_eq!(status.code(), Code::NotFound);- // The writer commits past the death record; the prune settles its mark.+ // A stub-create revival: same id and uuid, version above the death document's.+ set_row(death.version + 1, false).await;+ let revived = harness+ .client+ .get_person(get())+ .await+ .expect("the revival is served over the cached death document")+ .into_inner()+ .person+ .unwrap();+ assert_eq!(revived.version, death.version + 1);+ assert_eq!(revived.uuid, person_uuid);++ harness.cache.remove(partition, &key);+ let recovered = harness+ .client+ .get_person(get())+ .await+ .expect("the revival is served over the recovered death document")+ .into_inner()+ .person+ .unwrap();+ assert_eq!(recovered.version, death.version + 1);+let mark = harness.dirty_index.get(&key)@@ -609,21 +666,97 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {assert!(exhausted);assert_eq!(pruned.death_marks.len(), 1);drop_settled_death_documents(&harness.cache, &harness.locks, &pruned.death_marks).await;- assert!(- harness.cache.peek(partition, &key).is_none(),- "the settled death document leaves the cache"+ assert_eq!(+ harness+ .cache+ .peek(partition, &key)+ .map(|entry| entry.version),+ Some(death.version + 1));- let revived = harness+ sqlx::query("DELETE FROM lifecycle_op WHERE op_id = $1")+ .bind(op)+ .execute(&pool)+ .await+ .expect("cleanup op");+ sqlx::query("DELETE FROM posthog_person WHERE team_id = $1 AND id = $2")+ .bind(team_id as i32)+ .bind(person_id)+ .execute(&pool)+ .await+ .expect("cleanup person");+}++/// A property write and a version raise land on the revived person with no+/// read first, the path the persons store takes.+#[tokio::test]+async fn a_write_lands_on_the_revival_without_a_prior_read() {+ let pool = common::create_persons_pool().await;+ let mut harness = start_fence_harness(+ test_cached_person(),+ Some(PgFallback {+ pool: pool.clone(),+ table: "posthog_person".to_string(),+ lifecycle: LifecycleTables::new("lifecycle_op", "lifecycle_op_person"),+ }),+ )+ .await;+ let team_id = harness.team_id;+ let partition = harness.partition;+ let person_id = harness.person_id;+ let person_uuid = test_cached_person().uuid;+ let (op, death) = destroy_through_pg(&mut harness, &pool).await;+ let key = PersonCacheKey { team_id, person_id };++ sqlx::query(+ "INSERT INTO posthog_person \+ (id, team_id, uuid, properties, created_at, version, is_identified, is_deleted) \+ VALUES ($1, $2, $3::uuid, '{\"revived\": true}'::jsonb, now(), $4, false, false)",+ )+ .bind(person_id)+ .bind(team_id as i32)+ .bind(&person_uuid)+ .bind(death.version + 1)+ .execute(&pool)+ .await+ .expect("insert revived row");++ let written = harness.client- .get_person(get())+ .update_person_properties(with_partition(+ update_request(team_id, person_id),+ partition,+ )).await- .expect("the revival is served from PG after the settle")- .into_inner()- .person- .unwrap();- assert_eq!(revived.version, death.version + 1);- assert_eq!(revived.uuid, person_uuid);+ .expect("the write lands on the revived person")+ .into_inner();+ assert!(written.updated);+ let person = written.person.expect("the written person is returned");+ assert_eq!(person.uuid, person_uuid);+ assert!(person.version > death.version + 1);+ let properties: serde_json::Value =+ serde_json::from_slice(&person.properties).expect("properties parse");+ assert_eq!(properties["revived"], serde_json::json!(true));+ assert_eq!(properties["name"], serde_json::json!("after-fence"));++ let min_version = person.version + 10;+ let raised = harness+ .client+ .raise_person_version(with_partition(+ raise_request(team_id, person_id, min_version),+ partition,+ ))+ .await+ .expect("the raise lands on the revived person")+ .into_inner();+ assert!(raised.raised);+ assert_eq!(raised.person.unwrap().version, min_version);+ let cached = harness+ .cache+ .peek(partition, &key)+ .expect("the revived person stays cached");+ assert!(!cached.is_deleted);+ assert_eq!(cached.version, min_version);sqlx::query("DELETE FROM lifecycle_op WHERE op_id = $1").bind(op)diff --git a/rust/personhog-leader/tests/fencing_integration.rs b/rust/personhog-leader/tests/fencing_integration.rsindex 974acf0d1749..d05640131e56 100644--- a/rust/personhog-leader/tests/fencing_integration.rs+++ b/rust/personhog-leader/tests/fencing_integration.rs@@ -59,11 +59,16 @@ async fn read_committed_count(topic: &str) -> usize {let mut seen = 0;// Anything committed is available immediately; the quiet period only// has to outlast delivery, not a transaction timeout.- while let Ok(Ok(message)) =- tokio::time::timeout(Duration::from_millis(750), consumer.recv()).await- {- if message.payload().is_some() {- seen += 1;+ loop {+ match tokio::time::timeout(Duration::from_millis(750), consumer.recv()).await {+ Ok(Ok(message)) => {+ if message.payload().is_some() {+ seen += 1;+ }+ }+ Ok(Err(error)) if common::is_transient_broker_error(&error) => {}+ Ok(Err(error)) => panic!("kafka error reading {topic}: {error}"),+ Err(_quiet) => break,}}seendiff --git a/rust/personhog-leader/tests/integration.rs b/rust/personhog-leader/tests/integration.rsindex ad57d8207ed6..535807270f59 100644--- a/rust/personhog-leader/tests/integration.rs+++ b/rust/personhog-leader/tests/integration.rs@@ -1010,10 +1010,7 @@ async fn update_produces_person_state_to_kafka() {.unwrap();consumer.assign(&tpl).unwrap();- let msg = consumer- .poll(Duration::from_secs(5))- .expect("no message received on the routing partition")- .expect("kafka error");+ let msg = common::poll_record(&consumer, Duration::from_secs(5), "the routing partition");// Verify message keylet key = std::str::from_utf8(msg.key().unwrap()).unwrap();@@ -1257,10 +1254,7 @@ async fn e2e_update_produces_to_local_kafka() {.unwrap();consumer.assign(&tpl).unwrap();- let msg = consumer- .poll(Duration::from_secs(10))- .expect("no message received from local Kafka")- .expect("kafka error");+ let msg = common::poll_record(&consumer, Duration::from_secs(10), "local Kafka");let key = std::str::from_utf8(msg.key().unwrap()).unwrap();assert_eq!(key, "1:42");diff --git a/rust/personhog-test-harness/src/cli.rs b/rust/personhog-test-harness/src/cli.rsindex 4a12b862080c..e1f661154972 100644--- a/rust/personhog-test-harness/src/cli.rs+++ b/rust/personhog-test-harness/src/cli.rs@@ -210,7 +210,7 @@ pub struct GateArgs {#[arg(long, default_value = "personhog_person_tmp")]pub pg_target_table: String,- #[arg(long, default_value = "localhost:9092")]+ #[arg(long, default_value = "127.0.0.1:9092")]pub kafka_hosts: String,#[arg(long, default_value = "http://localhost:2379")]@@ -502,7 +502,7 @@ pub struct StackArgs {#[arg(long, default_value = "personhog_person_tmp")]pub pg_target_table: String,- #[arg(long, default_value = "localhost:9092")]+ #[arg(long, default_value = "127.0.0.1:9092")]pub kafka_hosts: String,#[arg(long, default_value = "http://localhost:2379")]