Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): serve a person revived over its cached death document (#128825)

Zach Browna105a0f8e066
diff --git a/rust/personhog-leader/src/service.rs b/rust/personhog-leader/src/service.rs
index 4e89fbe157d2..685d1808174c 100644
--- a/rust/personhog-leader/src/service.rs
+++ b/rust/personhog-leader/src/service.rs
@@ -100,6 +100,22 @@ struct Committed {
_inflight: InflightGuard,
}
+/// Registers the revival outcomes at zero. A revival under a standing
+/// death document is rare, so a series born at its first one would hide
+/// that one from `increase()`.
+fn register_revival_metrics() {
+ for outcome in [
+ "served",
+ "row_below_death",
+ "tombstoned",
+ "no_fallback",
+ "load_error",
+ ] {
+ counter!("personhog_leader_revivals_total", "outcome" => outcome).increment(0);
+ }
+ drop(histogram!("personhog_leader_person_load_duration_ms", "source" => "revival"));
+}
+
pub struct PersonHogLeaderService {
committer: Arc<Committer>,
cache: Arc<PartitionedCache>,
@@ -224,6 +240,7 @@ impl PersonHogLeaderService {
authority: Arc<AuthorityClock>,
emitted_versions: Arc<EmittedVersions>,
) -> Self {
+ register_revival_metrics();
let committer = Arc::new(Committer {
cache: Arc::clone(&cache),
dirty_index: Arc::clone(&dirty_index),
@@ -447,6 +464,73 @@ impl PersonHogLeaderService {
}
}
+ /// A live PG row above the death version can only be a stub-create
+ /// revival under the same id. Caller holds the per-key lock.
+ async fn serve_revival(
+ &self,
+ partition: u32,
+ key: &PersonCacheKey,
+ person: CachedPerson,
+ ) -> Result<CachedPerson, Status> {
+ if !person.is_deleted {
+ return Ok(person);
+ }
+ let Some(fallback) = &self.fallback else {
+ counter!("personhog_leader_revivals_total", "outcome" => "no_fallback").increment(1);
+ return Ok(person);
+ };
+ let started = Instant::now();
+ let result = load_person_from_pg(&fallback.pool, &fallback.table, key).await;
+ histogram!("personhog_leader_person_load_duration_ms", "source" => "revival")
+ .record(started.elapsed().as_secs_f64() * 1000.0);
+ match result {
+ Ok(Some(row)) if row.version > person.version => {
+ counter!("personhog_leader_revivals_total", "outcome" => "served").increment(1);
+ tracing::info!(
+ team_id = key.team_id,
+ person_id = key.person_id,
+ death_version = person.version,
+ version = row.version,
+ "serving a person revived in Postgres over its death document"
+ );
+ self.cache.put(partition, key.clone(), row.clone());
+ Ok(row)
+ }
+ Ok(Some(row)) => {
+ // The tombstone precedes the release, so a live row this
+ // low is a pre-death read; it must not revive the person.
+ counter!("personhog_leader_revivals_total", "outcome" => "row_below_death")
+ .increment(1);
+ tracing::warn!(
+ team_id = key.team_id,
+ person_id = key.person_id,
+ death_version = person.version,
+ version = row.version,
+ "a live Postgres row sits at or below the death document"
+ );
+ Ok(person)
+ }
+ Ok(None) => {
+ counter!("personhog_leader_revivals_total", "outcome" => "tombstoned").increment(1);
+ Ok(person)
+ }
+ Err(e) => {
+ counter!("personhog_leader_revivals_total", "outcome" => "load_error").increment(1);
+ counter!("personhog_leader_pg_fallback_errors_total").increment(1);
+ tracing::error!(
+ team_id = key.team_id,
+ person_id = key.person_id,
+ error = %e,
+ "PG revival check failed"
+ );
+ // Callers read not-found as gone, which this could not establish.
+ Err(Status::unavailable(
+ "could not check whether a destroyed person was revived; retry",
+ ))
+ }
+ }
+ }
+
/// Look up a person from cache, falling back to PG on miss.
/// Acquires a per-key lock.
async fn lookup_or_load(
@@ -454,12 +538,13 @@ impl PersonHogLeaderService {
partition: u32,
key: &PersonCacheKey,
) -> Result<CachedPerson, Status> {
- // Fast path: cache hit (no lock needed)
+ // Fast path: cache hit (no lock needed); a death document takes the locked path.
match self.cache.get(partition, key) {
- CacheLookup::Found(person) => {
+ CacheLookup::Found(person) if !person.is_deleted => {
Self::record_cache_hit();
return Ok(person);
}
+ CacheLookup::Found(_) => {}
CacheLookup::PartitionNotOwned => {
return Err(Status::failed_precondition(format!(
"partition {} not owned by this leader",
@@ -474,12 +559,13 @@ impl PersonHogLeaderService {
let _guard = mutex.lock().await;
// Double-check cache -- another request may have loaded it
- if let CacheLookup::Found(person) = self.cache.get(partition, key) {
+ let person = if let CacheLookup::Found(person) = self.cache.get(partition, key) {
Self::record_cache_hit();
- return Ok(person);
- }
-
- self.recover_or_load(partition, key).await
+ person
+ } else {
+ self.recover_or_load(partition, key).await?
+ };
+ self.serve_revival(partition, key, person).await
}
/// Look up a person from cache, falling back to PG on miss.
@@ -489,17 +575,20 @@ impl PersonHogLeaderService {
partition: u32,
key: &PersonCacheKey,
) -> Result<CachedPerson, Status> {
- match self.cache.get(partition, key) {
+ let person = match self.cache.get(partition, key) {
CacheLookup::Found(person) => {
Self::record_cache_hit();
- Ok(person)
+ person
}
- CacheLookup::PartitionNotOwned => Err(Status::failed_precondition(format!(
- "partition {} not owned by this leader",
- partition
- ))),
- CacheLookup::PersonNotFound => self.recover_or_load(partition, key).await,
- }
+ CacheLookup::PartitionNotOwned => {
+ return Err(Status::failed_precondition(format!(
+ "partition {} not owned by this leader",
+ partition
+ )))
+ }
+ CacheLookup::PersonNotFound => self.recover_or_load(partition, key).await?,
+ };
+ self.serve_revival(partition, key, person).await
}
/// The shared tail of every document write: refuse unapplyable records,
diff --git a/rust/personhog-leader/tests/common/mod.rs b/rust/personhog-leader/tests/common/mod.rs
index dd2b9622924b..8d1d867ae2a2 100644
--- a/rust/personhog-leader/tests/common/mod.rs
+++ b/rust/personhog-leader/tests/common/mod.rs
@@ -30,6 +30,9 @@ use personhog_coordination::types::{ChangelogConfig, ChangelogTarget};
use personhog_leader::coordination::{Capture, ChangelogCaptures};
use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
use rdkafka::client::DefaultClientContext;
+use rdkafka::consumer::BaseConsumer;
+use rdkafka::error::KafkaError;
+use rdkafka::message::BorrowedMessage;
use rdkafka::mocking::MockCluster;
use rdkafka::producer::{DefaultProducerContext, FutureProducer};
use rdkafka::types::RDKafkaErrorCode;
@@ -61,6 +64,41 @@ pub const WAIT_TIMEOUT: Duration = Duration::from_secs(10);
pub const POLL_INTERVAL: Duration = Duration::from_millis(100);
pub const NUM_PARTITIONS: u32 = 4;
+/// librdkafka reports a refused broker address and then reconnects on
+/// its own, for example when `localhost` resolves to `::1` first. A
+/// reader that treats the report as final fails on a broker about to
+/// answer, or counts a partition as drained that is not.
+pub fn is_transient_broker_error(error: &KafkaError) -> bool {
+ matches!(
+ error,
+ KafkaError::MessageConsumption(
+ RDKafkaErrorCode::BrokerTransportFailure | RDKafkaErrorCode::AllBrokersDown
+ )
+ )
+}
+
+/// One record from `consumer`, riding out transient broker errors until
+/// `wait` has passed. Panics on any other error and at the deadline.
+pub fn poll_record<'a>(
+ consumer: &'a BaseConsumer,
+ wait: Duration,
+ what: &str,
+) -> BorrowedMessage<'a> {
+ let deadline = std::time::Instant::now() + wait;
+ loop {
+ match consumer.poll(Duration::from_millis(200)) {
+ Some(Ok(message)) => return message,
+ Some(Err(error)) if is_transient_broker_error(&error) => {}
+ Some(Err(error)) => panic!("kafka error reading {what}: {error}"),
+ None => {}
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "no message received from {what} within {wait:?}"
+ );
+ }
+}
+
/// Find a person_id whose key hashes to `partition`, so a test can aim a
/// request at a specific partition while satisfying the leader's partition
/// validation.
diff --git a/rust/personhog-leader/tests/fence_integration.rs b/rust/personhog-leader/tests/fence_integration.rs
index 49475b68e3a4..dfb2087f247d 100644
--- a/rust/personhog-leader/tests/fence_integration.rs
+++ b/rust/personhog-leader/tests/fence_integration.rs
@@ -16,7 +16,7 @@ use common::{
use personhog_common::partitioning::partition_for_person;
use personhog_common::properties::jsonb_column_size;
use personhog_leader::cache::{
- CacheLookup, CachedPerson, DirtyIndex, PartitionedCache, PersonCacheKey,
+ CacheLookup, CachedPerson, CachedVersion, DirtyIndex, PartitionedCache, PersonCacheKey,
};
use personhog_leader::emitted::EmittedVersions;
use personhog_leader::fence::{
@@ -193,11 +193,12 @@ fn changelog_records(harness: &FenceHarness) -> Vec<Person> {
let mut records = Vec::new();
loop {
match consumer.poll(Duration::from_millis(300)) {
- Some(result) => {
- let msg = result.expect("kafka error");
+ Some(Ok(msg)) => {
records
.push(Person::decode(msg.payload().unwrap()).expect("decode changelog record"));
}
+ Some(Err(error)) if common::is_transient_broker_error(&error) => {}
+ Some(Err(error)) => panic!("kafka error: {error}"),
None if !records.is_empty() || std::time::Instant::now() >= deadline => break,
None => {}
}
@@ -488,21 +489,12 @@ async fn a_committed_release_produces_the_death_document_above_every_version() {
.expect("cleanup");
}
-/// While the death document's mark stands, reads answer not-found even if
-/// PG already holds a revived row. Once the mark settles, the document
-/// leaves the cache and the next read serves the revival from PG.
-#[tokio::test]
-async fn a_revival_is_served_once_the_death_documents_mark_settles() {
- let pool = common::create_persons_pool().await;
- let mut harness = start_fence_harness(
- test_cached_person(),
- Some(PgFallback {
- pool: pool.clone(),
- table: "posthog_person".to_string(),
- lifecycle: LifecycleTables::new("lifecycle_op", "lifecycle_op_person"),
- }),
- )
- .await;
+/// Fences and releases the harness person through a delete op marked in
+/// PG, leaving its death document cached. Returns the op and the document.
+async fn destroy_through_pg(
+ harness: &mut FenceHarness,
+ pool: &sqlx::PgPool,
+) -> (Uuid, CachedVersion) {
let team_id = harness.team_id;
let partition = harness.partition;
let person_id = harness.person_id;
@@ -515,7 +507,7 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {
)
.bind(op)
.bind(team_id as i32)
- .execute(&pool)
+ .execute(pool)
.await
.expect("insert op");
sqlx::query(
@@ -525,7 +517,7 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {
.bind(op)
.bind(team_id as i32)
.bind(person_id)
- .execute(&pool)
+ .execute(pool)
.await
.expect("insert mark");
@@ -564,21 +556,29 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {
.peek(partition, &key)
.expect("the death document is cached");
assert!(death.is_deleted);
+ (op, death)
+}
- // A stub-create revived the row in place: same id and uuid, version
- // above the death document's.
- sqlx::query(
- "INSERT INTO posthog_person \
- (id, team_id, uuid, properties, created_at, version, is_identified, is_deleted) \
- VALUES ($1, $2, $3::uuid, '{\"revived\": true}'::jsonb, now(), $4, false, false)",
+/// A live PG row above the death document's version is served over the
+/// cached document, over one recovered from the changelog, and past the settle.
+#[tokio::test]
+async fn a_revival_is_served_over_the_death_document() {
+ let pool = common::create_persons_pool().await;
+ let mut harness = start_fence_harness(
+ test_cached_person(),
+ Some(PgFallback {
+ pool: pool.clone(),
+ table: "posthog_person".to_string(),
+ lifecycle: LifecycleTables::new("lifecycle_op", "lifecycle_op_person"),
+ }),
)
- .bind(person_id)
- .bind(team_id as i32)
- .bind(&person_uuid)
- .bind(death.version + 1)
- .execute(&pool)
- .await
- .expect("insert revived row");
+ .await;
+ let team_id = harness.team_id;
+ let partition = harness.partition;
+ let person_id = harness.person_id;
+ let person_uuid = test_cached_person().uuid;
+ let (op, death) = destroy_through_pg(&mut harness, &pool).await;
+ let key = PersonCacheKey { team_id, person_id };
let get = || {
with_partition(
@@ -590,14 +590,71 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {
partition,
)
};
+ let set_row = |version: i64, is_deleted: bool| {
+ let pool = pool.clone();
+ let person_uuid = person_uuid.clone();
+ async move {
+ sqlx::query(
+ "INSERT INTO posthog_person \
+ (id, team_id, uuid, properties, created_at, version, is_identified, is_deleted) \
+ VALUES ($1, $2, $3::uuid, '{\"revived\": true}'::jsonb, now(), $4, false, $5) \
+ ON CONFLICT (team_id, id) DO UPDATE \
+ SET version = EXCLUDED.version, is_deleted = EXCLUDED.is_deleted",
+ )
+ .bind(person_id)
+ .bind(team_id as i32)
+ .bind(&person_uuid)
+ .bind(version)
+ .bind(is_deleted)
+ .execute(&pool)
+ .await
+ .expect("write row");
+ }
+ };
+
+ // The saga's own tombstone, as PG holds it between the unmap and the
+ // writer's settle.
+ set_row(death.version, true).await;
+ let status = harness
+ .client
+ .get_person(get())
+ .await
+ .expect_err("the death document answers over a tombstoned row");
+ assert_eq!(status.code(), Code::NotFound);
+
+ // A live row at the death version is pre-death state, not a revival.
+ set_row(death.version, false).await;
let status = harness
.client
.get_person(get())
.await
- .expect_err("the death document answers while its mark stands");
+ .expect_err("the death document still answers");
assert_eq!(status.code(), Code::NotFound);
- // The writer commits past the death record; the prune settles its mark.
+ // A stub-create revival: same id and uuid, version above the death document's.
+ set_row(death.version + 1, false).await;
+ let revived = harness
+ .client
+ .get_person(get())
+ .await
+ .expect("the revival is served over the cached death document")
+ .into_inner()
+ .person
+ .unwrap();
+ assert_eq!(revived.version, death.version + 1);
+ assert_eq!(revived.uuid, person_uuid);
+
+ harness.cache.remove(partition, &key);
+ let recovered = harness
+ .client
+ .get_person(get())
+ .await
+ .expect("the revival is served over the recovered death document")
+ .into_inner()
+ .person
+ .unwrap();
+ assert_eq!(recovered.version, death.version + 1);
+
let mark = harness
.dirty_index
.get(&key)
@@ -609,21 +666,97 @@ async fn a_revival_is_served_once_the_death_documents_mark_settles() {
assert!(exhausted);
assert_eq!(pruned.death_marks.len(), 1);
drop_settled_death_documents(&harness.cache, &harness.locks, &pruned.death_marks).await;
- assert!(
- harness.cache.peek(partition, &key).is_none(),
- "the settled death document leaves the cache"
+ assert_eq!(
+ harness
+ .cache
+ .peek(partition, &key)
+ .map(|entry| entry.version),
+ Some(death.version + 1)
);
- let revived = harness
+ sqlx::query("DELETE FROM lifecycle_op WHERE op_id = $1")
+ .bind(op)
+ .execute(&pool)
+ .await
+ .expect("cleanup op");
+ sqlx::query("DELETE FROM posthog_person WHERE team_id = $1 AND id = $2")
+ .bind(team_id as i32)
+ .bind(person_id)
+ .execute(&pool)
+ .await
+ .expect("cleanup person");
+}
+
+/// A property write and a version raise land on the revived person with no
+/// read first, the path the persons store takes.
+#[tokio::test]
+async fn a_write_lands_on_the_revival_without_a_prior_read() {
+ let pool = common::create_persons_pool().await;
+ let mut harness = start_fence_harness(
+ test_cached_person(),
+ Some(PgFallback {
+ pool: pool.clone(),
+ table: "posthog_person".to_string(),
+ lifecycle: LifecycleTables::new("lifecycle_op", "lifecycle_op_person"),
+ }),
+ )
+ .await;
+ let team_id = harness.team_id;
+ let partition = harness.partition;
+ let person_id = harness.person_id;
+ let person_uuid = test_cached_person().uuid;
+ let (op, death) = destroy_through_pg(&mut harness, &pool).await;
+ let key = PersonCacheKey { team_id, person_id };
+
+ sqlx::query(
+ "INSERT INTO posthog_person \
+ (id, team_id, uuid, properties, created_at, version, is_identified, is_deleted) \
+ VALUES ($1, $2, $3::uuid, '{\"revived\": true}'::jsonb, now(), $4, false, false)",
+ )
+ .bind(person_id)
+ .bind(team_id as i32)
+ .bind(&person_uuid)
+ .bind(death.version + 1)
+ .execute(&pool)
+ .await
+ .expect("insert revived row");
+
+ let written = harness
.client
- .get_person(get())
+ .update_person_properties(with_partition(
+ update_request(team_id, person_id),
+ partition,
+ ))
.await
- .expect("the revival is served from PG after the settle")
- .into_inner()
- .person
- .unwrap();
- assert_eq!(revived.version, death.version + 1);
- assert_eq!(revived.uuid, person_uuid);
+ .expect("the write lands on the revived person")
+ .into_inner();
+ assert!(written.updated);
+ let person = written.person.expect("the written person is returned");
+ assert_eq!(person.uuid, person_uuid);
+ assert!(person.version > death.version + 1);
+ let properties: serde_json::Value =
+ serde_json::from_slice(&person.properties).expect("properties parse");
+ assert_eq!(properties["revived"], serde_json::json!(true));
+ assert_eq!(properties["name"], serde_json::json!("after-fence"));
+
+ let min_version = person.version + 10;
+ let raised = harness
+ .client
+ .raise_person_version(with_partition(
+ raise_request(team_id, person_id, min_version),
+ partition,
+ ))
+ .await
+ .expect("the raise lands on the revived person")
+ .into_inner();
+ assert!(raised.raised);
+ assert_eq!(raised.person.unwrap().version, min_version);
+ let cached = harness
+ .cache
+ .peek(partition, &key)
+ .expect("the revived person stays cached");
+ assert!(!cached.is_deleted);
+ assert_eq!(cached.version, min_version);
sqlx::query("DELETE FROM lifecycle_op WHERE op_id = $1")
.bind(op)
diff --git a/rust/personhog-leader/tests/fencing_integration.rs b/rust/personhog-leader/tests/fencing_integration.rs
index 974acf0d1749..d05640131e56 100644
--- a/rust/personhog-leader/tests/fencing_integration.rs
+++ b/rust/personhog-leader/tests/fencing_integration.rs
@@ -59,11 +59,16 @@ async fn read_committed_count(topic: &str) -> usize {
let mut seen = 0;
// Anything committed is available immediately; the quiet period only
// has to outlast delivery, not a transaction timeout.
- while let Ok(Ok(message)) =
- tokio::time::timeout(Duration::from_millis(750), consumer.recv()).await
- {
- if message.payload().is_some() {
- seen += 1;
+ loop {
+ match tokio::time::timeout(Duration::from_millis(750), consumer.recv()).await {
+ Ok(Ok(message)) => {
+ if message.payload().is_some() {
+ seen += 1;
+ }
+ }
+ Ok(Err(error)) if common::is_transient_broker_error(&error) => {}
+ Ok(Err(error)) => panic!("kafka error reading {topic}: {error}"),
+ Err(_quiet) => break,
}
}
seen
diff --git a/rust/personhog-leader/tests/integration.rs b/rust/personhog-leader/tests/integration.rs
index ad57d8207ed6..535807270f59 100644
--- a/rust/personhog-leader/tests/integration.rs
+++ b/rust/personhog-leader/tests/integration.rs
@@ -1010,10 +1010,7 @@ async fn update_produces_person_state_to_kafka() {
.unwrap();
consumer.assign(&tpl).unwrap();
- let msg = consumer
- .poll(Duration::from_secs(5))
- .expect("no message received on the routing partition")
- .expect("kafka error");
+ let msg = common::poll_record(&consumer, Duration::from_secs(5), "the routing partition");
// Verify message key
let key = std::str::from_utf8(msg.key().unwrap()).unwrap();
@@ -1257,10 +1254,7 @@ async fn e2e_update_produces_to_local_kafka() {
.unwrap();
consumer.assign(&tpl).unwrap();
- let msg = consumer
- .poll(Duration::from_secs(10))
- .expect("no message received from local Kafka")
- .expect("kafka error");
+ let msg = common::poll_record(&consumer, Duration::from_secs(10), "local Kafka");
let key = std::str::from_utf8(msg.key().unwrap()).unwrap();
assert_eq!(key, "1:42");
diff --git a/rust/personhog-test-harness/src/cli.rs b/rust/personhog-test-harness/src/cli.rs
index 4a12b862080c..e1f661154972 100644
--- a/rust/personhog-test-harness/src/cli.rs
+++ b/rust/personhog-test-harness/src/cli.rs
@@ -210,7 +210,7 @@ pub struct GateArgs {
#[arg(long, default_value = "personhog_person_tmp")]
pub pg_target_table: String,
- #[arg(long, default_value = "localhost:9092")]
+ #[arg(long, default_value = "127.0.0.1:9092")]
pub kafka_hosts: String,
#[arg(long, default_value = "http://localhost:2379")]
@@ -502,7 +502,7 @@ pub struct StackArgs {
#[arg(long, default_value = "personhog_person_tmp")]
pub pg_target_table: String,
- #[arg(long, default_value = "localhost:9092")]
+ #[arg(long, default_value = "127.0.0.1:9092")]
pub kafka_hosts: String,
#[arg(long, default_value = "http://localhost:2379")]