:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
mod common;
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::{Duration, Instant};
use common_ingestion_warnings::throttle::{WarningThrottle, DEFAULT_THROTTLE_PERIOD};
use dashmap::DashMap;
use common::{
create_leader_client, create_local_kafka_producer, create_test_kafka,
create_test_kafka_with_partitions, person_id_for_partition, seed_person, seed_test_changelog,
served_captures, start_coordinator, start_leader_pod, start_leader_pod_with_lease_ttl,
start_leader_with_pg_fallback, start_router, target, test_cached_person, test_recovery,
test_store, test_warming_config, wait_for_condition, KAFKA_BOOTSTRAP, NUM_PARTITIONS,
POLL_INTERVAL, WAIT_TIMEOUT,
};
use personhog_common::partitioning::partition_for_person;
use personhog_coordination::pod::HandoffHandler;
use personhog_coordination::strategy::StickyBalancedStrategy;
use personhog_coordination::types::ChangelogConfig;
use personhog_leader::cache::{
CacheLookup, CachedPerson, DirtyIndex, DirtyMark, PartitionedCache, PersonCacheKey,
};
use personhog_leader::coordination::LeaderHandoffHandler;
use personhog_leader::inflight::InflightTracker;
use personhog_leader::service::{PersonHogLeaderService, PropertySizeLimits};
use personhog_leader::warming::WarmClientPools;
use personhog_leader::warnings::WarningsProducer;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeader;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;
use personhog_proto::personhog::types::v1::{
GetPersonRequest, Person, UpdatePersonPropertiesRequest,
};
use prost::Message;
use rdkafka::consumer::{BaseConsumer, Consumer};
use rdkafka::{ClientConfig, Message as KafkaMessage, TopicPartitionList};
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
use tonic::transport::Server;
use tonic::Request;
/// Wrap a request with the `x-partition` metadata the leader reads in place
/// of a body field, matching how the router forwards to the leader.
fn with_partition<T>(req: T, partition: u32) -> Request<T> {
let mut request = Request::new(req);
request
.metadata_mut()
.insert("x-partition", partition.to_string().parse().unwrap());
request
}
/// Build a `GetPerson` request carrying the partition in `x-partition`
/// metadata, matching how the router forwards strong reads to the leader.
fn leader_get_request(team_id: i64, person_id: i64, partition: u32) -> Request<GetPersonRequest> {
with_partition(
GetPersonRequest {
team_id,
person_id,
read_options: None,
},
partition,
)
}
// ============================================================
// Test 1: Full lifecycle via etcd coordination
// ============================================================
#[tokio::test]
async fn service_accepts_requests_after_coordination_warmup() {
let store = test_store("leader-warmup").await;
let cancel = CancellationToken::new();
let _topic = seed_test_changelog(&store).await;
let _coord = start_coordinator(
Arc::clone(&store),
Arc::new(StickyBalancedStrategy),
cancel.clone(),
);
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
// Single pod gets all partitions
let pod = start_leader_pod(Arc::clone(&store), "leader-0", 1 << 20, cancel.clone()).await;
// Wait for all partitions to be assigned
let check_store = Arc::clone(&store);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let store = Arc::clone(&check_store);
async move {
let assignments = store.list_assignments().await.unwrap_or_default();
assignments.len() == NUM_PARTITIONS as usize
&& assignments.iter().all(|a| a.owner == "leader-0")
}
})
.await;
// Wait for handoffs to complete (warmup finished)
let check_store = Arc::clone(&store);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let store = Arc::clone(&check_store);
async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
})
.await;
// Wait for all partitions to be warmed in the cache (may lag behind etcd state)
let check_cache = Arc::clone(&pod.cache);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let cache = Arc::clone(&check_cache);
async move { (0..NUM_PARTITIONS).all(|p| cache.has_partition(p)) }
})
.await;
// Seed a person into partition 0
let person = test_cached_person();
seed_person(&pod.cache, 0, person.clone());
// gRPC get_person should succeed
let mut client = create_leader_client(pod.leader_addr).await;
let response = client
.get_person(leader_get_request(1, 42, 0))
.await
.unwrap();
let proto_person = response.into_inner().person.unwrap();
assert_eq!(proto_person.id, 42);
assert_eq!(proto_person.team_id, 1);
// Update should succeed
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Updated"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await
.unwrap();
let result = response.into_inner();
assert!(result.updated);
assert_eq!(result.person.unwrap().version, 2);
// Read back should reflect the update
let response = client
.get_person(leader_get_request(1, 42, 0))
.await
.unwrap();
let proto_person = response.into_inner().person.unwrap();
assert_eq!(proto_person.version, 2);
let props: serde_json::Value = serde_json::from_slice(&proto_person.properties).unwrap();
assert_eq!(props["name"], "Updated");
assert_eq!(props["email"], "[email protected]");
cancel.cancel();
}
// ============================================================
// Test 2: Unowned partition returns FailedPrecondition
// (no etcd needed)
// ============================================================
#[tokio::test]
async fn unowned_partition_returns_failed_precondition() {
// Create service + cache directly, no coordination
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
// One set for both: a second would fence the first at the broker.
let fenced =
common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
// Request for unowned partition → FailedPrecondition
let result = client.get_person(leader_get_request(1, 42, 0)).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::FailedPrecondition);
// Write to unowned partition → FailedPrecondition
let result = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: vec![],
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::FailedPrecondition);
// Manually warm partition → now NotFound (partition exists, no data)
cache.create_partition(0);
let result = client.get_person(leader_get_request(1, 42, 0)).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);
cancel.cancel();
}
// ============================================================
// Test 2b: Requests without x-partition metadata fail closed
// (no etcd needed)
// ============================================================
/// The partition arrives only via `x-partition` metadata, stamped by the
/// router. A request without it is misrouted or malformed and must be
/// rejected with INVALID_ARGUMENT rather than served against a guessed
/// partition — even when the person exists and its partition is warm.
#[tokio::test]
async fn missing_partition_metadata_returns_invalid_argument() {
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
// One set for both: a second would fence the first at the broker.
let fenced =
common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
// Warm the partition and seed the person so the only failure mode
// left is the missing metadata itself.
cache.create_partition(0);
seed_person(&cache, 0, test_cached_person());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
let result = client
.get_person(GetPersonRequest {
team_id: 1,
person_id: 42,
read_options: None,
})
.await;
let status = result.expect_err("read without x-partition must fail closed");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert!(status.message().contains("x-partition"));
let result = client
.update_person_properties(UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: vec![],
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
})
.await;
let status = result.expect_err("write without x-partition must fail closed");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert!(status.message().contains("x-partition"));
cancel.cancel();
}
// ============================================================
// Test 2c: x-partition must match the partition derived from the
// request's own key (no etcd needed)
// ============================================================
/// The leader validates the router's routing decision against the decoded
/// body: `x-partition` must equal `partition_for_person(team_id,
/// person_id)`. A mismatch means a client stamped wrong routing-key
/// headers or the hash implementations diverged — serving it would read or
/// write through the wrong partition's cache, so it must be rejected even
/// when the named partition is warm and the person exists there.
#[tokio::test]
async fn mismatched_partition_metadata_returns_failed_precondition() {
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
// One set for both: a second would fence the first at the broker.
let fenced =
common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
// Key (1, 42) hashes to some true partition; pick a different one and
// warm + seed it so the only failure mode left is the mismatch itself.
let true_partition = partition_for_person(1, 42, NUM_PARTITIONS);
let wrong_partition = (true_partition + 1) % NUM_PARTITIONS;
cache.create_partition(wrong_partition);
seed_person(&cache, wrong_partition, test_cached_person());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
let result = client
.get_person(leader_get_request(1, 42, wrong_partition))
.await;
let status = result.expect_err("read with mismatched x-partition must fail closed");
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
assert!(status.message().contains("does not match"));
let result = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: vec![],
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
wrong_partition,
))
.await;
let status = result.expect_err("write with mismatched x-partition must fail closed");
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
assert!(status.message().contains("does not match"));
cancel.cancel();
}
// ============================================================
// Test 2b: Post-drain write fencing
// (no etcd needed)
// ============================================================
/// Once the old owner has drained a partition, every router has acked the
/// freeze — so a later write can only come from a router serving with a
/// stale table (an expired lease it hasn't noticed, a missed freeze).
/// Accepting it would produce to Kafka past the HWM that warming
/// snapshots, silently losing the write from the new owner's cache. After
/// draining, the old owner must reject writes for the partition while
/// continuing to serve reads: the frozen state remains the latest until
/// cutover. Releasing the partition clears the fence with it.
#[tokio::test]
async fn writes_fenced_after_drain_reads_still_served() {
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
let inflight = Arc::new(InflightTracker::new());
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
let recovery = test_recovery(KAFKA_BOOTSTRAP);
// One set for both: a second would fence the first at the broker.
let fenced =
common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
let captures = served_captures(&_mock_cluster.topic, NUM_PARTITIONS);
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::clone(&inflight),
Arc::clone(&captures),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
// The handler shares the cache, inflight tracker, dirty index, and
// recovery pool with the service, exactly as main.rs wires them.
let warming = test_warming_config("fence-pod", KAFKA_BOOTSTRAP);
let pools = Arc::new(WarmClientPools::new(
&warming.kafka,
"fence-pod",
&warming.writer_consumer_group,
));
let changelog = common::live_changelog(&_mock_cluster.topic);
let handler = LeaderHandoffHandler::new(
Arc::clone(&cache),
Arc::clone(&inflight),
Arc::clone(&dirty_index),
warming,
Arc::new(DashMap::new()),
None,
changelog,
Arc::clone(&captures),
pools,
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(0);
seed_person(&cache, 0, test_cached_person());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
let update = |email: &str| {
with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({ "email": email })).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
)
};
// Pre-drain: writes flow normally.
let response = client
.update_person_properties(update("[email protected]"))
.await
.unwrap();
assert!(response.into_inner().updated);
// Drain the partition, as the pod does on observing Draining.
handler.drain_partition_inflight(0).await.unwrap();
// Post-drain: writes must be fenced…
let status = client
.update_person_properties(update("[email protected]"))
.await
.expect_err("write after drain must be rejected");
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
// …while reads keep serving the frozen, still-latest state.
let response = client
.get_person(leader_get_request(1, 42, 0))
.await
.expect("reads must keep serving after drain");
let props: serde_json::Value =
serde_json::from_slice(&response.into_inner().person.unwrap().properties).unwrap();
assert_eq!(props["email"], "[email protected]");
// Releasing clears the fence along with the partition. The warm is
// stood in for, epoch included, since warming needs a changelog this
// test never writes.
handler.release_partition(0).await.unwrap();
handler
.capture_changelog(
0,
&ChangelogConfig::new(&_mock_cluster.topic, NUM_PARTITIONS),
)
.await;
cache.create_partition(0);
seed_person(&cache, 0, test_cached_person());
fenced
.acquire(0, &target(&_mock_cluster.topic))
.await
.expect("the re-warm takes the epoch");
let response = client
.update_person_properties(update("[email protected]"))
.await
.expect("writes must flow again after release + re-warm");
assert!(response.into_inner().updated);
cancel.cancel();
}
/// The fence must go up before the drain starts waiting on inflight
/// handlers — fencing only after `wait_until_empty` returns would leave a
/// window where a write lands between the inflight count hitting zero and
/// the DrainedAck being written, advancing the Kafka HWM past what
/// warming will read. With an inflight write held open, new writes must
/// already be rejected while the drain is still waiting.
#[tokio::test]
async fn drain_fences_before_waiting_on_inflight() {
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
let inflight = Arc::new(InflightTracker::new());
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
let recovery = test_recovery(KAFKA_BOOTSTRAP);
// One set for both: a second would fence the first at the broker.
let fenced =
common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
let captures = served_captures(&_mock_cluster.topic, NUM_PARTITIONS);
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::clone(&inflight),
Arc::clone(&captures),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
let warming = test_warming_config("fence-race-pod", KAFKA_BOOTSTRAP);
let pools = Arc::new(WarmClientPools::new(
&warming.kafka,
"fence-race-pod",
&warming.writer_consumer_group,
));
let changelog = common::live_changelog(&_mock_cluster.topic);
let handler = Arc::new(LeaderHandoffHandler::new(
Arc::clone(&cache),
Arc::clone(&inflight),
Arc::clone(&dirty_index),
warming,
Arc::new(DashMap::new()),
None,
changelog,
Arc::clone(&captures),
pools,
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
));
cache.create_partition(0);
seed_person(&cache, 0, test_cached_person());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
// Hold a simulated in-flight write so the drain parks in
// wait_until_empty rather than completing immediately.
let guard = inflight.begin(0);
let drain_handler = Arc::clone(&handler);
let drain = tokio::spawn(async move { drain_handler.drain_partition_inflight(0).await });
// Give the drain a moment to start waiting.
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
!drain.is_finished(),
"drain must wait for the held inflight"
);
// A write arriving mid-drain must already be fenced.
let status = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(
&serde_json::json!({"email": "[email protected]"}),
)
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await
.expect_err("write during drain must be rejected");
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
// Releasing the held write lets the drain complete.
drop(guard);
tokio::time::timeout(Duration::from_secs(5), drain)
.await
.expect("drain must complete once inflight reaches zero")
.expect("drain task must not panic")
.expect("drain must succeed");
cancel.cancel();
}
// ============================================================
// Test 3: Release partition stops serving
// ============================================================
#[tokio::test]
async fn release_partition_stops_serving() {
let store = test_store("leader-release").await;
let cancel = CancellationToken::new();
let _topic = seed_test_changelog(&store).await;
let _coord = start_coordinator(
Arc::clone(&store),
Arc::new(StickyBalancedStrategy),
cancel.clone(),
);
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
// Start pod 1 — gets all partitions
let pod1 = start_leader_pod(Arc::clone(&store), "leader-0", 1 << 20, cancel.clone()).await;
let check_store = Arc::clone(&store);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let store = Arc::clone(&check_store);
async move {
let assignments = store.list_assignments().await.unwrap_or_default();
let handoffs = store.list_handoffs().await.unwrap_or_default();
assignments.len() == NUM_PARTITIONS as usize
&& assignments.iter().all(|a| a.owner == "leader-0")
&& handoffs.is_empty()
}
})
.await;
// Wait for cache to reflect assignments (may lag behind etcd state)
let check_cache = Arc::clone(&pod1.cache);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let cache = Arc::clone(&check_cache);
async move { (0..NUM_PARTITIONS).all(|p| cache.has_partition(p)) }
})
.await;
// Seed a person into partition 0
seed_person(&pod1.cache, 0, test_cached_person());
// Verify get_person works on pod 1
let mut client1 = create_leader_client(pod1.leader_addr).await;
let response = client1
.get_person(leader_get_request(1, 42, 0))
.await
.unwrap();
assert_eq!(response.into_inner().person.unwrap().id, 42);
// Start pod 2 — triggers rebalance
let pod2 = start_leader_pod(Arc::clone(&store), "leader-1", 1 << 20, cancel.clone()).await;
// Wait for balanced assignment and handoffs to settle
let check_store = Arc::clone(&store);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let store = Arc::clone(&check_store);
async move {
let assignments = store.list_assignments().await.unwrap_or_default();
let handoffs = store.list_handoffs().await.unwrap_or_default();
let has_both = assignments.iter().any(|a| a.owner == "leader-0")
&& assignments.iter().any(|a| a.owner == "leader-1");
assignments.len() == NUM_PARTITIONS as usize && has_both && handoffs.is_empty()
}
})
.await;
// Find a partition that moved from pod 1 to pod 2
let assignments = store.list_assignments().await.unwrap();
let moved_partition = assignments
.iter()
.find(|a| a.owner == "leader-1")
.expect("pod 2 should own at least one partition")
.partition;
// Wait for caches to reflect the rebalance (may lag behind etcd state)
let check_cache1 = Arc::clone(&pod1.cache);
let check_cache2 = Arc::clone(&pod2.cache);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let c1 = Arc::clone(&check_cache1);
let c2 = Arc::clone(&check_cache2);
async move { !c1.has_partition(moved_partition) && c2.has_partition(moved_partition) }
})
.await;
// A key that actually hashes to the moved partition, so the request
// passes the leader's partition validation.
let moved_person_id = person_id_for_partition(1, moved_partition);
// Pod 1: released partition → FailedPrecondition
let result = client1
.get_person(leader_get_request(1, moved_person_id, moved_partition))
.await;
assert!(result.is_err());
assert_eq!(
result.unwrap_err().code(),
tonic::Code::FailedPrecondition,
"pod 1 should reject requests for released partition {moved_partition}"
);
// Pod 2: warmed partition but empty cache → NotFound
let mut client2 = create_leader_client(pod2.leader_addr).await;
let result = client2
.get_person(leader_get_request(1, moved_person_id, moved_partition))
.await;
assert!(result.is_err());
assert_eq!(
result.unwrap_err().code(),
tonic::Code::NotFound,
"pod 2 should have partition {moved_partition} warmed but no data"
);
cancel.cancel();
}
// ============================================================
// Test 4: Rewarm after pod crash
// ============================================================
#[tokio::test]
async fn rewarm_after_pod_crash() {
let store = test_store("leader-rewarm").await;
let cancel = CancellationToken::new();
let _topic = seed_test_changelog(&store).await;
let _coord = start_coordinator(
Arc::clone(&store),
Arc::new(StickyBalancedStrategy),
cancel.clone(),
);
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
// Pod 1 with short lease (crash detectable quickly)
let pod1_cancel = CancellationToken::new();
let _pod1 = start_leader_pod_with_lease_ttl(
Arc::clone(&store),
"leader-0",
1 << 20,
2,
pod1_cancel.clone(),
)
.await;
// Pod 2 (long-lived)
let pod2 = start_leader_pod(Arc::clone(&store), "leader-1", 1 << 20, cancel.clone()).await;
// Wait for balanced assignment
let check_store = Arc::clone(&store);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let store = Arc::clone(&check_store);
async move {
let assignments = store.list_assignments().await.unwrap_or_default();
let handoffs = store.list_handoffs().await.unwrap_or_default();
let has_both = assignments.iter().any(|a| a.owner == "leader-0")
&& assignments.iter().any(|a| a.owner == "leader-1");
assignments.len() == NUM_PARTITIONS as usize && has_both && handoffs.is_empty()
}
})
.await;
// Kill pod 1
pod1_cancel.cancel();
// Wait for pod 1 to disappear and all partitions to move to pod 2
let check_store = Arc::clone(&store);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let store = Arc::clone(&check_store);
async move {
let pods = store.list_pods().await.unwrap_or_default();
let assignments = store.list_assignments().await.unwrap_or_default();
let handoffs = store.list_handoffs().await.unwrap_or_default();
pods.len() == 1
&& assignments.len() == NUM_PARTITIONS as usize
&& assignments.iter().all(|a| a.owner == "leader-1")
&& handoffs.is_empty()
}
})
.await;
// Wait for pod 2 to re-warm all partitions (may lag behind etcd state)
let check_cache = Arc::clone(&pod2.cache);
wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
let cache = Arc::clone(&check_cache);
async move { (0..NUM_PARTITIONS).all(|p| cache.has_partition(p)) }
})
.await;
// Partitions are warm but empty → NotFound
let mut client2 = create_leader_client(pod2.leader_addr).await;
let result = client2.get_person(leader_get_request(1, 42, 0)).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);
cancel.cancel();
}
// ============================================================
// Test 5: Successful update produces person state to Kafka
// (no etcd needed)
// ============================================================
#[tokio::test]
async fn update_produces_person_state_to_kafka() {
// The changelog must land on the exact Kafka partition the request's
// `x-partition` named — warming rebuilds a routing partition's cache by
// consuming the same-numbered Kafka partition, so key-hash placement
// (whose partitioner config could diverge from the router's murmur2)
// is not acceptable. The key (team 1, person 2) murmur2-hashes to
// partition 2 (so it passes the leader's partition validation) but
// librdkafka's default crc32 partitioner would place it on partition 0,
// keeping misplacement observable: the consumer below reads only
// partition 2 and must find the message there.
const PERSON_ID: i64 = 2;
let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
assert_eq!(routing_partition, 2, "test key must map to partition 2");
let mock_cluster = create_test_kafka_with_partitions(4).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(routing_partition);
let person = CachedPerson {
id: PERSON_ID,
..test_cached_person()
};
seed_person(&cache, routing_partition, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
// Perform an update
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Kafka Test"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap();
let result = response.into_inner();
assert!(result.updated);
assert_eq!(result.person.unwrap().version, 2);
// Consume only the routing partition — finding the message here proves
// the explicit-partition produce, not just delivery.
let consumer: BaseConsumer = ClientConfig::new()
.set("bootstrap.servers", mock_cluster.bootstrap_servers())
.set("group.id", "test-consumer")
.create()
.expect("failed to create consumer");
let mut tpl = TopicPartitionList::new();
tpl.add_partition_offset(
&mock_cluster.topic,
routing_partition as i32,
rdkafka::Offset::Beginning,
)
.unwrap();
consumer.assign(&tpl).unwrap();
let msg = common::poll_record(&consumer, Duration::from_secs(5), "the routing partition");
// Verify message key
let key = std::str::from_utf8(msg.key().unwrap()).unwrap();
assert_eq!(key, "1:2");
// Verify payload is a valid Person proto with updated state
let person = Person::decode(msg.payload().unwrap()).unwrap();
assert_eq!(person.id, PERSON_ID);
assert_eq!(person.team_id, 1);
assert_eq!(person.version, 2);
let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
assert_eq!(props["name"], "Kafka Test");
assert_eq!(props["email"], "[email protected]");
cancel.cancel();
}
// ============================================================
// Test 6: a changelog write this pod cannot land leaves the cache
// unchanged and returns an error (no etcd needed)
// ============================================================
#[tokio::test]
async fn kafka_produce_failure_leaves_cache_unchanged() {
let cache = Arc::new(PartitionedCache::new(1 << 20));
let mock_cluster = create_test_kafka().await;
let fenced =
common::fenced_producers_acquired(&mock_cluster.topic, NUM_PARTITIONS as i32).await;
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(0);
let person = test_cached_person();
seed_person(&cache, 0, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
// A successor takes the partition's transactional id, which is what
// a changelog write this pod cannot land looks like.
let successor =
common::fenced_producers_acquired(&mock_cluster.topic, NUM_PARTITIONS as i32).await;
let mut client = create_leader_client(addr).await;
// Update should fail because Kafka produce fails
let result = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Should Rollback"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await;
assert!(result.is_err());
assert_eq!(
result.unwrap_err().code(),
tonic::Code::FailedPrecondition,
"a fenced pod answers in the ownership vocabulary, so the router re-resolves"
);
// Cache was never updated since the produce failed before the cache write
let cache_key = PersonCacheKey {
team_id: 1,
person_id: 42,
};
let CacheLookup::Found(cached) = cache.get(0, &cache_key) else {
panic!("expected original person in cache");
};
assert_eq!(cached.version, 1);
assert_eq!(
cached.parse_properties().unwrap()["email"],
"[email protected]"
);
assert!(cached.parse_properties().unwrap().get("name").is_none());
// This pod re-takes the epoch, as the convergence heal does.
drop(successor);
fenced.expect_self_fence(0);
fenced
.acquire(0, &target(&mock_cluster.topic))
.await
.expect("this pod re-takes the fence");
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "After Recovery"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await
.unwrap();
let result = response.into_inner();
assert!(result.updated);
// Two, not three: a record the broker refused never existed, so its
// version is free again.
assert_eq!(
result.person.unwrap().version,
2,
"a version the broker refused is not spent"
);
cancel.cancel();
}
// ============================================================
// Test 7: E2E - update produces to local Kafka and is consumable
// Requires local Kafka (127.0.0.1:9092) and etcd running.
// ============================================================
#[tokio::test]
async fn e2e_update_produces_to_local_kafka() {
let owned = common::owned_broker_topic(NUM_PARTITIONS as i32).await;
let topic = owned.topic.clone();
let cache = Arc::new(PartitionedCache::new(1 << 20));
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&topic, NUM_PARTITIONS as i32).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(0);
let person = test_cached_person();
seed_person(&cache, 0, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
// Perform an update via gRPC
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "E2E Test"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await
.unwrap();
let result = response.into_inner();
assert!(result.updated);
assert_eq!(result.person.unwrap().version, 2);
// Consume from local Kafka and verify the message
let consumer: BaseConsumer = ClientConfig::new()
.set("bootstrap.servers", KAFKA_BOOTSTRAP)
.set("group.id", format!("e2e-test-{}", uuid::Uuid::new_v4()))
.create()
.expect("failed to create local Kafka consumer");
// From the beginning: counting back from the end lands on the
// transaction's commit marker, which no consumer is handed.
let mut tpl = TopicPartitionList::new();
tpl.add_partition_offset(&topic, 0, rdkafka::Offset::Beginning)
.unwrap();
consumer.assign(&tpl).unwrap();
let msg = common::poll_record(&consumer, Duration::from_secs(10), "local Kafka");
let key = std::str::from_utf8(msg.key().unwrap()).unwrap();
assert_eq!(key, "1:42");
let person = Person::decode(msg.payload().unwrap()).unwrap();
assert_eq!(person.id, 42);
assert_eq!(person.team_id, 1);
assert_eq!(person.version, 2);
let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
assert_eq!(props["name"], "E2E Test");
assert_eq!(props["email"], "[email protected]");
cancel.cancel();
}
// ============================================================
// Test 8: PG fallback on cache miss
// Requires local Postgres.
// ============================================================
#[tokio::test]
async fn pg_fallback_loads_person_on_cache_miss() {
let cancel = CancellationToken::new();
let (addr, cache, _mock_cluster) = start_leader_with_pg_fallback(cancel.clone()).await;
// Seed a person under this test's own team id. Sampling an arbitrary
// existing row instead would race concurrently-running tests that
// delete their transient rows between our sample and the fallback read.
let pool = common::create_persons_pool().await;
let team_id: i32 = 99_061;
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(team_id)
.execute(&pool)
.await
.unwrap();
let (person_id,): (i64,) = sqlx::query_as(
"INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id)
VALUES (now(), '{\"plan\": \"pro\"}'::jsonb, false, gen_random_uuid(), 1, $1)
RETURNING id",
)
.bind(team_id)
.fetch_one(&pool)
.await
.unwrap();
// Warm the key's own partition (the cache is empty — no persons seeded)
let partition = partition_for_person(team_id as i64, person_id, NUM_PARTITIONS);
cache.create_partition(partition);
let mut client = create_leader_client(addr).await;
// First call: cache miss → PG fallback → loads and caches the person
let response = client
.get_person(leader_get_request(team_id as i64, person_id, partition))
.await
.unwrap();
let person = response.into_inner().person.unwrap();
assert_eq!(person.id, person_id);
assert_eq!(person.team_id, team_id as i64);
// Verify person is now cached
let key = PersonCacheKey {
team_id: team_id as i64,
person_id,
};
assert!(
matches!(cache.get(partition, &key), CacheLookup::Found(_)),
"person should be cached after PG fallback"
);
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(team_id)
.execute(&pool)
.await
.unwrap();
cancel.cancel();
}
/// Rows written by other services can hold numerics whose PG-expanded
/// rendering serde_json rejects even though JSON.parse reads them fine
/// (JS `Number.MAX_VALUE` is the canonical case — a common "unlimited"
/// sentinel). The fallback must load such rows the way JS would — rounding
/// representable values, clamping beyond-f64 garbage — never panic or
/// leave the person permanently unloadable.
#[tokio::test]
async fn pg_fallback_reads_numerics_the_leaders_parser_rejects() {
let pool = common::create_persons_pool().await;
let team_id: i32 = 99_060;
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(team_id)
.execute(&pool)
.await
.unwrap();
// Written exactly as Node's pg driver would: e-notation literals that
// PG stores as full-precision numerics and renders back expanded.
let row: (i64,) = sqlx::query_as(
"INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id)
VALUES (now(),
'{\"credits\": 1.7976931348623157e+308, \"overflow\": 2e308, \"plan\": \"pro\"}'::jsonb,
false, gen_random_uuid(), 1, $1)
RETURNING id",
)
.bind(team_id)
.fetch_one(&pool)
.await
.unwrap();
let key = PersonCacheKey {
team_id: team_id as i64,
person_id: row.0,
};
let person = personhog_leader::pg::load_person_from_pg(&pool, "posthog_person", &key)
.await
.expect("load must not fail")
.expect("person exists");
// The representable sentinel reads as the exact double JS reads;
// beyond-f64 garbage clamps instead of poisoning the row; neighbors
// are untouched.
assert_eq!(
person.parse_properties().unwrap()["credits"]
.as_f64()
.unwrap(),
f64::MAX
);
assert_eq!(
person.parse_properties().unwrap()["overflow"]
.as_f64()
.unwrap(),
1e307
);
assert_eq!(person.parse_properties().unwrap()["plan"], "pro");
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(team_id)
.execute(&pool)
.await
.unwrap();
}
// ============================================================
// Test 9: PG fallback returns NotFound for non-existent person
// ============================================================
#[tokio::test]
async fn pg_fallback_returns_not_found_for_missing_person() {
let cancel = CancellationToken::new();
let (addr, cache, _mock_cluster) = start_leader_with_pg_fallback(cancel.clone()).await;
let partition = partition_for_person(99999, 99999999, NUM_PARTITIONS);
cache.create_partition(partition);
let mut client = create_leader_client(addr).await;
// Query a person that doesn't exist in PG
let result = client
.get_person(leader_get_request(99999, 99999999, partition))
.await;
assert!(result.is_err());
assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);
cancel.cancel();
}
// ============================================================
// Test 10: Update triggers PG fallback on cache miss then applies changes
// ============================================================
#[tokio::test]
async fn update_triggers_pg_fallback_then_applies_changes() {
let cancel = CancellationToken::new();
let (addr, cache, _mock_cluster) = start_leader_with_pg_fallback(cancel.clone()).await;
// Seed a person under this test's own team id (see the cache-miss
// test above for why sampling an arbitrary row is not safe).
let pool = common::create_persons_pool().await;
let team_id: i32 = 99_062;
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(team_id)
.execute(&pool)
.await
.unwrap();
let (person_id,): (i64,) = sqlx::query_as(
"INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id)
VALUES (now(), '{\"plan\": \"pro\"}'::jsonb, false, gen_random_uuid(), 1, $1)
RETURNING id",
)
.bind(team_id)
.fetch_one(&pool)
.await
.unwrap();
let partition = partition_for_person(team_id as i64, person_id, NUM_PARTITIONS);
cache.create_partition(partition);
let mut client = create_leader_client(addr).await;
// Update a person not in cache — should load from PG then apply
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: team_id as i64,
person_id,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({
"pg_fallback_test": "it_works"
}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
partition,
))
.await
.unwrap();
let result = response.into_inner();
assert!(result.updated);
let updated_person = result.person.unwrap();
assert_eq!(updated_person.id, person_id);
let props: serde_json::Value = serde_json::from_slice(&updated_person.properties).unwrap();
assert_eq!(props["pg_fallback_test"], "it_works");
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(team_id)
.execute(&pool)
.await
.unwrap();
cancel.cancel();
}
// ============================================================
// Dirty-index recovery: eviction of a person whose latest state
// exists only in the changelog must recover from Kafka, never PG
// ============================================================
#[tokio::test]
async fn evicted_dirty_person_recovers_from_changelog() {
const PERSON_ID: i64 = 2;
let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
let mock_cluster = create_test_kafka_with_partitions(4).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
// Recovery reads from the same mock broker the update produces to.
// Deliberately no PG pool: a recovery path that (wrongly) fell back to
// PG would return NotFound and fail the final assertion.
let recovery = test_recovery(&mock_cluster.bootstrap_servers());
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(routing_partition);
let person = CachedPerson {
id: PERSON_ID,
..test_cached_person()
};
seed_person(&cache, routing_partition, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
// The update produces the v2 record and marks the dirty index.
client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Recovered"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap();
assert_eq!(dirty_index.len(), 1, "the acked write must be marked dirty");
// Simulate eviction of the freshly written entry.
let key = PersonCacheKey {
team_id: 1,
person_id: PERSON_ID,
};
cache.remove(routing_partition, &key);
// The read must return the updated state, recovered from the changelog
// record at the marked offset.
let response = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id: PERSON_ID,
read_options: None,
},
routing_partition,
))
.await
.unwrap();
let person = response.into_inner().person.unwrap();
assert_eq!(person.version, 2);
let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
assert_eq!(props["name"], "Recovered");
cancel.cancel();
}
// ============================================================
// Dirty-index recovery failure: when the changelog fetch cannot
// complete, the person is unavailable — not silently stale
// ============================================================
#[tokio::test]
async fn dirty_person_with_failed_recovery_is_unavailable_not_stale() {
const PERSON_ID: i64 = 2;
let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
let mock_cluster = create_test_kafka_with_partitions(4).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
let recovery = test_recovery(&mock_cluster.bootstrap_servers());
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(routing_partition);
let person = CachedPerson {
id: PERSON_ID,
..test_cached_person()
};
seed_person(&cache, routing_partition, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Pending"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap();
let key = PersonCacheKey {
team_id: 1,
person_id: PERSON_ID,
};
// Point the mark past the HWM so the recovery fetch cannot complete
// (mark only accepts newer offsets, so bump well beyond the record's).
let mark = dirty_index.get(&key).unwrap();
dirty_index.mark(
key.clone(),
DirtyMark {
version: mark.version,
offset: mark.offset + 1_000,
partition: mark.partition,
stream: mark.stream.clone(),
is_deleted: mark.is_deleted,
},
);
cache.remove(routing_partition, &key);
let status = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id: PERSON_ID,
read_options: None,
},
routing_partition,
))
.await
.unwrap_err();
assert_eq!(status.code(), tonic::Code::Unavailable);
cancel.cancel();
}
// ============================================================
// Dirty-index bound: a full index sheds writes for new persons
// rather than acking writes it cannot track
// ============================================================
#[tokio::test]
async fn writes_shed_when_dirty_index_is_full() {
let partition: u32 = 2;
let person_a = person_id_for_partition(1, partition);
let person_b = (person_a + 1..)
.find(|pid| partition_for_person(1, *pid, NUM_PARTITIONS) == partition)
.expect("a second person maps to the same partition");
let mock_cluster = create_test_kafka_with_partitions(4).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
// Capacity 1: the first person's mark fills the index.
let dirty_index = Arc::new(DirtyIndex::new(1));
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::clone(&dirty_index),
test_recovery(&mock_cluster.bootstrap_servers()),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(partition);
seed_person(
&cache,
partition,
CachedPerson {
id: person_a,
..test_cached_person()
},
);
seed_person(
&cache,
partition,
CachedPerson {
id: person_b,
..test_cached_person()
},
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
let update_for = |person_id: i64, value: &str| UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({ "k": value })).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
};
// First person fills the index.
client
.update_person_properties(with_partition(update_for(person_a, "v1"), partition))
.await
.expect("first person's write is admitted");
// The same person stays admitted at capacity — updating its mark does
// not grow the index.
client
.update_person_properties(with_partition(update_for(person_a, "v2"), partition))
.await
.expect("marked person's writes stay admitted at capacity");
// A new person cannot be tracked: the write sheds instead of acking.
let status = client
.update_person_properties(with_partition(update_for(person_b, "v1"), partition))
.await
.unwrap_err();
assert_eq!(status.code(), tonic::Code::ResourceExhausted);
cancel.cancel();
}
// ============================================================
// Recovery version tripwire: a record whose version disagrees
// with the acked mark must fail the read, never be served
// ============================================================
#[tokio::test]
async fn recovery_fails_when_record_version_disagrees_with_the_mark() {
const PERSON_ID: i64 = 2;
let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
let mock_cluster = create_test_kafka_with_partitions(4).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
let recovery = test_recovery(&mock_cluster.bootstrap_servers());
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(routing_partition);
let person = CachedPerson {
id: PERSON_ID,
..test_cached_person()
};
seed_person(&cache, routing_partition, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Tripwire"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap();
let key = PersonCacheKey {
team_id: 1,
person_id: PERSON_ID,
};
// Corrupt the mark's version while keeping the true offset — `mark`
// only replaces newer offsets, so clear the partition and re-insert.
// The seek now lands on the right record, but its version disagrees.
let mark = dirty_index.get(&key).unwrap();
dirty_index.clear_partition(routing_partition);
dirty_index.mark(
key.clone(),
DirtyMark {
version: mark.version + 1,
..mark
},
);
cache.remove(routing_partition, &key);
let status = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id: PERSON_ID,
read_options: None,
},
routing_partition,
))
.await
.unwrap_err();
assert_eq!(status.code(), tonic::Code::Unavailable);
cancel.cancel();
}
// ============================================================
// Recovery consumer pooling: repeated recoveries on a partition
// reuse one consumer, repositioning forward and backward
// ============================================================
#[tokio::test]
async fn recovery_reuses_the_partition_consumer_across_fetches() {
let partition: u32 = 2;
let person_a = person_id_for_partition(1, partition);
let person_b = (person_a + 1..)
.find(|pid| partition_for_person(1, *pid, NUM_PARTITIONS) == partition)
.expect("a second person maps to the same partition");
let mock_cluster = create_test_kafka_with_partitions(4).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
let recovery = test_recovery(&mock_cluster.bootstrap_servers());
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(partition);
for person_id in [person_a, person_b] {
seed_person(
&cache,
partition,
CachedPerson {
id: person_id,
..test_cached_person()
},
);
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let client = create_leader_client(addr).await;
let update_for = |person_id: i64, value: &str| UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({ "who": value })).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
};
// A's record lands at offset 0, B's at offset 1.
for (person_id, value) in [(person_a, "a"), (person_b, "b")] {
client
.clone()
.update_person_properties(with_partition(update_for(person_id, value), partition))
.await
.unwrap();
}
let recover = |person_id: i64| {
let mut client = client.clone();
async move {
let response = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id,
read_options: None,
},
partition,
))
.await
.unwrap();
let person = response.into_inner().person.unwrap();
let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
props["who"].as_str().unwrap().to_string()
}
};
let evict = |person_id: i64| {
cache.remove(
partition,
&PersonCacheKey {
team_id: 1,
person_id,
},
)
};
// All three recoveries share the partition's pooled consumer: the
// first fetches at offset 0, the second repositions forward, and the
// third repositions backward past a record the consumer already read.
// A consumer that fails to reposition serves the wrong record (key
// mismatch) or times out — either fails the unwraps above.
evict(person_a);
assert_eq!(recover(person_a).await, "a");
evict(person_b);
assert_eq!(recover(person_b).await, "b");
evict(person_a);
assert_eq!(recover(person_a).await, "a");
cancel.cancel();
}
/// A client disconnect mid-recovery drops the request future at an await
/// point. Checkout is an RAII guard, so the cancelled fetch's consumer
/// must come home: without Drop-based return, every cancellation leaked
/// the consumer while freeing its permit, and pool_size cancellations
/// left permits pointing at an empty pool — a panic under the pool mutex
/// that poisoned it and disabled recovery until restart.
#[tokio::test]
async fn cancelled_recovery_returns_its_consumer_to_the_pool() {
use common::test_kafka_config;
use personhog_leader::recovery::{ChangelogRecovery, RecoveryConfig};
use rdkafka::producer::FutureRecord;
let mock_cluster = create_test_kafka().await;
let producer = create_local_kafka_producer().await;
let mut kafka = test_kafka_config();
kafka.kafka_hosts = mock_cluster.bootstrap_servers();
// pool_size 1: a single leaked consumer exhausts the pool.
let recovery = ChangelogRecovery::new(RecoveryConfig {
kafka,
pod_name: "cancel-pod".to_string(),
recv_timeout: Duration::from_secs(2),
pool_size: 1,
})
.unwrap();
let key = PersonCacheKey {
team_id: 1,
person_id: 7,
};
let mark = DirtyMark {
version: 1,
offset: 0,
partition: 0,
stream: personhog_leader::cache::Stream {
cluster: None,
topic: std::sync::Arc::from(mock_cluster.topic.as_str()),
partition: 0,
},
is_deleted: false,
};
// Nothing is produced yet, so each fetch parks awaiting the record;
// cancel it mid-park, more times than the pool holds consumers.
for _ in 0..3 {
let cancelled = tokio::time::timeout(
Duration::from_millis(50),
recovery.fetch_person_at(&mark, &key),
)
.await;
assert!(
cancelled.is_err(),
"the fetch must still be parked when cancelled"
);
}
// The pool must still function end to end: produce the sought record
// and the next fetch — checkout included — succeeds.
let person = Person {
id: 7,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id: 1,
properties: serde_json::to_vec(&serde_json::json!({"k": "v"})).unwrap(),
properties_last_updated_at: Vec::new(),
properties_last_operation: Vec::new(),
created_at: 1_700_000_000,
version: 1,
is_identified: false,
is_user_id: None,
last_seen_at: None,
is_deleted: false,
};
let payload = person.encode_to_vec();
producer
.send(
FutureRecord::to(&mock_cluster.topic)
.key("1:7")
.partition(0)
.payload(&payload),
Duration::from_secs(5),
)
.await
.expect("produce");
let recovered = recovery
.fetch_person_at(&mark, &key)
.await
.expect("the pool must survive cancellations");
assert_eq!(recovered.id, 7);
}
#[tokio::test]
async fn a_mark_on_another_cluster_recovers_from_its_broker() {
use personhog_coordination::types::ClusterEndpoint;
use rdkafka::producer::FutureRecord;
let Some(secondary) = common::secondary_kafka_bootstrap() else {
return;
};
let old_topic = common::owned_broker_topic_on(&secondary, 1).await;
let producer = common::create_kafka_producer_on(&secondary).await;
let person = Person {
id: 7,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id: 1,
properties: b"{}".to_vec(),
properties_last_updated_at: Vec::new(),
properties_last_operation: Vec::new(),
created_at: 1_700_000_000,
version: 1,
is_identified: false,
is_user_id: None,
last_seen_at: None,
is_deleted: false,
};
producer
.send(
FutureRecord::to(&old_topic.topic)
.key("1:7")
.partition(0)
.payload(&person.encode_to_vec()),
Duration::from_secs(5),
)
.await
.expect("produce");
let recovery = test_recovery(KAFKA_BOOTSTRAP);
let mark = DirtyMark {
version: person.version,
offset: 0,
partition: 0,
stream: personhog_leader::cache::Stream {
cluster: Some(std::sync::Arc::new(ClusterEndpoint {
name: "old-cluster".to_string(),
hosts: secondary,
tls: false,
})),
topic: std::sync::Arc::from(old_topic.topic.as_str()),
partition: 0,
},
is_deleted: false,
};
let key = PersonCacheKey {
team_id: 1,
person_id: 7,
};
let recovered = recovery
.fetch_person_at(&mark, &key)
.await
.expect("recovers from the mark's cluster");
assert_eq!(recovered.id, 7);
}
// ============================================================
// Admission-time property size enforcement, mirroring the Node
// pipeline's policy: an update that would newly push a within-limit
// row over the ceiling is rejected; a row already over it is
// remediated (existing properties trimmed, the update discarded) —
// so every acked record is applyable by the writer verbatim.
// ============================================================
#[tokio::test]
async fn oversize_updates_are_rejected_and_oversized_rows_remediated() {
const PERSON_ID: i64 = 9;
let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
let mock_cluster = create_test_kafka_with_partitions(4).await;
// This test reads its warnings back, so they must be its own: on a
// shared topic it would collect whichever two arrived first.
let warnings_topic = common::owned_broker_topic(1).await;
let cache = Arc::new(PartitionedCache::new(1 << 20));
let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
let recovery = test_recovery(&mock_cluster.bootstrap_servers());
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&mock_cluster.topic, NUM_PARTITIONS),
Arc::clone(&dirty_index),
recovery,
PropertySizeLimits::new(655360, 524288),
// Burst 2 so the trim and reject below — same team, same type —
// both clear the throttle; the third enforcement action then
// exercises suppression.
common::local_warnings_producer(
&warnings_topic.topic,
WarningThrottle::new(DEFAULT_THROTTLE_PERIOD, NonZeroU32::new(2).unwrap()),
)
.await,
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(routing_partition);
let person = CachedPerson {
id: PERSON_ID,
..test_cached_person()
};
seed_person(&cache, routing_partition, person);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let cancel = CancellationToken::new();
let token = cancel.child_token();
tokio::spawn(async move {
Server::builder()
.add_service(PersonHogLeaderServer::new(service))
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
token.cancelled(),
)
.await
.unwrap();
});
tokio::time::sleep(Duration::from_millis(10)).await;
let mut client = create_leader_client(addr).await;
// New violation: the seeded person is within limits and this update
// would push the merged state over the ceiling — rejected outright,
// nothing stored, matching the Node pipeline's
// attempt_to_violate_limit path.
let big = "x".repeat(400_000);
let err = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({
"email": "[email protected]", "custom_a": big, "custom_b": big,
}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.expect_err("a newly violating update must be rejected");
assert_eq!(err.code(), tonic::Code::InvalidArgument);
assert!(err.message().contains("size limit"));
assert!(
!err.message().contains("xxx"),
"the error must carry sizes, never property values"
);
// Rejection leaves no residue: the rejected update's keys are absent
// from a strong read, and a normal update still works.
let read = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id: PERSON_ID,
read_options: None,
},
routing_partition,
))
.await
.unwrap()
.into_inner();
let read_props: serde_json::Value =
serde_json::from_slice(&read.person.unwrap().properties).unwrap();
assert!(!read_props.as_object().unwrap().contains_key("custom_a"));
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "after"})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap()
.into_inner();
assert!(response.updated);
// Remediation: a row already over the ceiling (legacy rows, and rows
// from the Node writer during dual-write) is healed on its next
// update — existing properties trimmed alphabetically to the target
// and the triggering update's changes discarded, matching the Node
// pipeline's existing_record_violates_limit path. custom_a goes
// (alphabetically first) and custom_b, which alone fits the target,
// survives.
const OVERSIZED_PERSON_ID: i64 = 10;
let oversized_partition = partition_for_person(1, OVERSIZED_PERSON_ID, NUM_PARTITIONS);
if oversized_partition != routing_partition {
cache.create_partition(oversized_partition);
}
seed_person(
&cache,
oversized_partition,
CachedPerson {
id: OVERSIZED_PERSON_ID,
properties: serde_json::to_vec(&serde_json::json!({
"email": "[email protected]", "custom_a": big, "custom_b": big,
}))
.unwrap(),
..test_cached_person()
},
);
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: OVERSIZED_PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "discarded"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
oversized_partition,
))
.await
.unwrap()
.into_inner();
assert!(response.updated);
let stored: serde_json::Value =
serde_json::from_slice(&response.person.unwrap().properties).unwrap();
let stored = stored.as_object().unwrap();
assert_eq!(stored["email"], "[email protected]", "protected properties survive");
assert!(
!stored.contains_key("custom_a"),
"custom_a must be trimmed (alphabetically first)"
);
assert!(stored.contains_key("custom_b"));
assert!(
!stored.contains_key("name"),
"the triggering update's changes are discarded, as in Node"
);
// The stored state IS the remediated state: a strong read agrees
// with the update response.
let read = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id: OVERSIZED_PERSON_ID,
read_options: None,
},
oversized_partition,
))
.await
.unwrap()
.into_inner();
let read_props: serde_json::Value =
serde_json::from_slice(&read.person.unwrap().properties).unwrap();
assert!(!read_props.as_object().unwrap().contains_key("custom_a"));
// Unremediable: the stored row is over the ceiling and its protected
// property alone exceeds the trim target — nothing can be trimmed,
// so the update is rejected. (Its warning lands after the throttle
// burst of 2 and is suppressed; the error is asserted directly.)
const UNREMEDIABLE_PERSON_ID: i64 = 12;
let unremediable_partition = partition_for_person(1, UNREMEDIABLE_PERSON_ID, NUM_PARTITIONS);
if unremediable_partition != routing_partition && unremediable_partition != oversized_partition
{
cache.create_partition(unremediable_partition);
}
let huge_email = "e".repeat(700_000);
seed_person(
&cache,
unremediable_partition,
CachedPerson {
id: UNREMEDIABLE_PERSON_ID,
properties: serde_json::to_vec(&serde_json::json!({ "email": huge_email })).unwrap(),
..test_cached_person()
},
);
let err = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: UNREMEDIABLE_PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "x"})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
unremediable_partition,
))
.await
.expect_err("an unremediable row's update must be rejected");
assert_eq!(err.code(), tonic::Code::InvalidArgument);
assert!(err.message().contains("trim target"));
assert!(
!err.message().contains("eee"),
"the error must carry sizes, never property values"
);
// The first two enforcement actions emitted in-product warnings from
// the leader: one rejection, one remediation trim.
let consumer: BaseConsumer = ClientConfig::new()
.set("bootstrap.servers", mock_cluster.bootstrap_servers())
.set("group.id", "warnings-check")
.create()
.unwrap();
let mut tpl = TopicPartitionList::new();
tpl.add_partition_offset(&warnings_topic.topic, 0, rdkafka::Offset::Beginning)
.unwrap();
consumer.assign(&tpl).unwrap();
let mut warnings = Vec::new();
let deadline = Instant::now() + Duration::from_secs(10);
while warnings.len() < 2 && Instant::now() < deadline {
if let Some(Ok(message)) = consumer.poll(Duration::from_millis(200)) {
let payload: serde_json::Value =
serde_json::from_slice(message.payload().unwrap()).unwrap();
assert_eq!(payload["type"], "person_properties_size_violation");
assert_eq!(payload["source"], "personhog-leader");
// The pipeline's canonical shape: a ClickHouse-format
// timestamp, and category/severity in details (the v2 table
// materializes columns from those exact keys).
let timestamp = payload["timestamp"].as_str().unwrap();
assert!(
timestamp.len() == 23 && timestamp.as_bytes()[10] == b' ',
"timestamp must be ClickHouse-format, got {timestamp}"
);
let details: serde_json::Value =
serde_json::from_str(payload["details"].as_str().unwrap()).unwrap();
assert_eq!(details["category"], "size");
assert_eq!(details["severity"], "error");
assert_eq!(details["pipelineStep"], "personhog_admission");
assert!(
details["personId"].as_str().unwrap().contains('-'),
"personId is the uuid"
);
warnings.push(payload["details"].as_str().unwrap().to_string());
}
}
assert_eq!(
warnings.len(),
2,
"expected a trim warning and a reject warning"
);
assert!(warnings.iter().any(|w| w.contains("trimmed")));
assert!(warnings.iter().any(|w| w.contains("rejected")));
// Team 1's (team, type) budget is exhausted: another enforcement
// action still rejects, but its warning is suppressed. A fresh
// team's warning still emits, and — the warnings topic being a
// single partition — arriving as the very next message proves the
// suppressed one was never produced.
const TEAM_2_PERSON_ID: i64 = 11;
let team2_partition = partition_for_person(2, TEAM_2_PERSON_ID, NUM_PARTITIONS);
if team2_partition != routing_partition {
cache.create_partition(team2_partition);
}
seed_person(
&cache,
team2_partition,
CachedPerson {
id: TEAM_2_PERSON_ID,
team_id: 2,
..test_cached_person()
},
);
let oversized_update = serde_json::to_vec(&serde_json::json!({
"email": "[email protected]", "custom_y": big, "custom_z": big,
}))
.unwrap();
client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: oversized_update.clone(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.expect_err("throttling gates the warning, not the enforcement");
client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 2,
person_id: TEAM_2_PERSON_ID,
event_name: "$set".to_string(),
set_properties: oversized_update,
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
team2_partition,
))
.await
.expect_err("team 2's newly violating update is rejected too");
let deadline = Instant::now() + Duration::from_secs(10);
let next = loop {
match consumer.poll(Duration::from_millis(200)) {
Some(Ok(message)) => break message,
_ => assert!(Instant::now() < deadline, "team 2's warning must arrive"),
}
};
let payload: serde_json::Value = serde_json::from_slice(next.payload().unwrap()).unwrap();
assert_eq!(
payload["team_id"], 2,
"team 1's third warning must be suppressed by the throttle"
);
// NUL bytes are sanitized at admission (`\u{0000}` → `\u{FFFD}`,
// matching the Node pipeline): Postgres jsonb refuses NUL, so an
// unsanitized record would be unapplyable by the writer. The stored
// state — response and strong read alike — carries the sanitized form.
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({
"nul\u{0000}key": "nul\u{0000}value",
}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap()
.into_inner();
assert!(response.updated);
let stored: serde_json::Value =
serde_json::from_slice(&response.person.unwrap().properties).unwrap();
let stored = stored.as_object().unwrap();
assert_eq!(stored["nul\u{FFFD}key"], "nul\u{FFFD}value");
assert!(!stored.contains_key("nul\u{0000}key"));
let read = client
.get_person(with_partition(
GetPersonRequest {
team_id: 1,
person_id: PERSON_ID,
read_options: None,
},
routing_partition,
))
.await
.unwrap()
.into_inner();
let read_person = read.person.unwrap();
let read_props: serde_json::Value = serde_json::from_slice(&read_person.properties).unwrap();
assert_eq!(read_props["nul\u{FFFD}key"], "nul\u{FFFD}value");
let version_after_nul = read_person.version;
// Resending the identical NUL-bearing update must hit the no-change
// fast path: inputs are sanitized before diffing, so the raw NUL form
// compares equal to its stored U+FFFD form instead of producing a
// fresh record and version bump on every repeat.
let response = client
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: PERSON_ID,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({
"nul\u{0000}key": "nul\u{0000}value",
}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
routing_partition,
))
.await
.unwrap()
.into_inner();
assert!(
!response.updated,
"a sanitized repeat must be detected as a no-op"
);
assert_eq!(
response.person.unwrap().version,
version_after_nul,
"a no-op repeat must not bump the version"
);
cancel.cancel();
}
// ============================================================
// A version this pod emitted without hearing the outcome is spent
// ============================================================
/// A cancelled request, or a commit whose fate stayed unknown, can leave a
/// record on the changelog that this pod never learned about. The cache
/// still holds the version from before that write, so the next update for
/// the same person would derive the same number again — and the writer's
/// strict version guard keeps whichever of the two records arrived first,
/// discarding the other. When the discarded one is the acked write, the
/// acknowledgement was a lie.
///
/// Driving the RPC rather than the derivation: the floor existing is not
/// the property that matters, the write path consulting it is.
#[tokio::test]
async fn an_unresolved_version_is_never_reused() {
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
// One set for both: a second would fence the first at the broker.
let fenced =
common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000));
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
Arc::clone(&emitted_versions),
);
cache.create_partition(0);
let person = test_cached_person();
let cached_version = person.version;
seed_person(&cache, 0, person);
// The state a write that was never answered for leaves behind: the
// cache is untouched, but versions up to this one may already exist.
let spent = cached_version + 4;
emitted_versions.raise_for_test(
0,
PersonCacheKey {
team_id: 1,
person_id: 42,
},
spent,
);
let response = service
.update_person_properties(with_partition(
UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"name": "Updated"}))
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
},
0,
))
.await
.expect("the update itself must still succeed");
assert_eq!(
response.into_inner().person.unwrap().version,
spent + 1,
"the write must derive past every version this pod already emitted, \
not merely past the one its cache happens to hold"
);
}
// ============================================================
// The fenced write path, driven through the RPC
// ============================================================
/// Nothing constructed the service with fencing on, so its entire fenced
/// arm was unreachable: a `panic!` at the top of it left the whole suite
/// green. That arm decides, for every failure mode, whether the write's
/// version is handed back for reuse — and handing back a version whose
/// record may be on the changelog is the acked-write loss the floors
/// exist to prevent.
///
/// This drives the real RPC against a real broker with fencing enabled,
/// and asserts the two answers that matter: a fenced partition bounces,
/// and the version it consumed is *not* reused afterwards.
#[tokio::test]
async fn a_fenced_write_that_bounces_does_not_hand_its_version_back() {
let topic = format!("fenced_svc_{}", uuid::Uuid::new_v4().simple());
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
let fenced = Arc::new(common::test_fenced_producers());
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
);
cache.create_partition(0);
let person = test_cached_person();
let base_version = person.version;
seed_person(&cache, 0, person);
fenced
.acquire(0, &target(&topic))
.await
.expect("take the partition's fence");
// Each call must change something, or the handler takes its
// no-change fast path and never reaches the produce at all.
let update = |n: i64| {
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"n": n})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
// A healthy fenced write goes through the transaction window.
let first = service
.update_person_properties(update(1))
.await
.expect("a fenced write with a held fence must succeed");
assert_eq!(first.into_inner().person.unwrap().version, base_version + 1);
// Give the fence up beneath the service: the next write has no
// producer and must answer as unowned so the router re-resolves.
fenced.release(0);
let bounced = service
.update_person_properties(update(2))
.await
.expect_err("a write with no fence must not be acked");
assert_eq!(
bounced.code(),
tonic::Code::FailedPrecondition,
"an unowned partition is the router's bounce class, not a retryable failure"
);
// The bounced write never reached the broker, so its version is free
// to derive again — the next successful write must land exactly one
// past the first, not two.
fenced
.acquire(0, &target(&topic))
.await
.expect("re-take the fence");
let third = service
.update_person_properties(update(3))
.await
.expect("a re-acquired fence writes again");
assert_eq!(
third.into_inner().person.unwrap().version,
base_version + 2,
"a write that provably never left the pod must not burn its version"
);
}
/// A window fenced with its own outcome unknown answers two questions at
/// once, and the handler has to honour both: the router gets the
/// ownership bounce, and the version stays spent because the commit may
/// have landed on an attempt librdkafka re-issued internally. Handing the
/// version back would put a second record at the same number behind one
/// that may be committed, and the writer's first-wins guard then discards
/// whichever arrived second — which is the acked one.
///
/// The cached entry stays. Evicting it resolves nothing, since recovery
/// reads the last *marked* offset — the previous write that did succeed —
/// and it would answer NOT_FOUND outright once that mark is pruned on a
/// pod with no fallback pool configured.
#[tokio::test]
async fn a_fence_with_an_unknown_outcome_keeps_both_its_version_and_its_cache_entry() {
for (staged, expected) in [
(
personhog_leader::fencing::FencedProduceError::FencedUncertain("staged".to_string()),
tonic::Code::FailedPrecondition,
),
(
personhog_leader::fencing::FencedProduceError::Indeterminate("staged".to_string()),
tonic::Code::Unknown,
),
] {
an_unknown_outcome_keeps_its_version(staged, expected).await;
}
}
/// Both doubt arms carry the same obligation: the record may be in the
/// log, so the version stays spent and the cached entry stays put. They
/// differ only in what the caller is told — an ownership bounce when the
/// partition also moved, plain doubt when it did not.
async fn an_unknown_outcome_keeps_its_version(
staged: personhog_leader::fencing::FencedProduceError,
expected: tonic::Code,
) {
let topic = format!("fenced_uncertain_{}", uuid::Uuid::new_v4().simple());
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
let fenced = Arc::new(common::test_fenced_producers());
let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000));
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
Arc::clone(&fenced),
common::live_authority(),
Arc::clone(&emitted_versions),
);
cache.create_partition(0);
let person = test_cached_person();
let base_version = person.version;
seed_person(&cache, 0, person);
fenced
.acquire(0, &target(&topic))
.await
.expect("take the partition's fence");
let cache_key = PersonCacheKey {
team_id: 1,
person_id: 42,
};
let update = |n: i64| {
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"n": n})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
service
.update_person_properties(update(1))
.await
.expect("a fenced write with a held fence must succeed");
// The outcome a broker fault inside a transaction would produce.
fenced.stage_failure_for_test(0, staged);
let bounced = service
.update_person_properties(update(2))
.await
.expect_err("an uncertain fence must not be acked");
assert_eq!(
bounced.code(),
expected,
"the caller's answer: {}",
bounced.message()
);
assert_eq!(
emitted_versions.floor_for(0, &cache_key, 0),
base_version + 2,
"a version whose record may be committed must stay spent"
);
assert!(
matches!(
cache.get(0, &cache_key),
personhog_leader::cache::CacheLookup::Found(_)
),
"the entry must survive, or a pruned mark leaves the person unreadable"
);
}
/// The scalar fields carry merge semantics, not assignment: identification
/// only ever ORs true (and counts as a change on its own — an $identify
/// with no property diffs must still produce a record), while last-seen
/// only ever advances and never earns a record by itself.
#[tokio::test]
async fn scalar_fields_merge_rather_than_assign() {
const HOUR_MS: i64 = 3_600_000;
let topic = format!("scalar_merge_{}", uuid::Uuid::new_v4().simple());
let cache = Arc::new(PartitionedCache::new(1 << 20));
let _mock_cluster = create_test_kafka().await;
let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000));
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
served_captures(&topic, NUM_PARTITIONS),
Arc::new(DirtyIndex::new(1_000_000)),
test_recovery(KAFKA_BOOTSTRAP),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
common::fenced_producers_acquired(&topic, NUM_PARTITIONS as i32).await,
common::live_authority(),
Arc::clone(&emitted_versions),
);
cache.create_partition(0);
seed_person(&cache, 0, test_cached_person());
let update = |set: serde_json::Value, identified: Option<bool>, seen: Option<i64>| {
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id: 1,
person_id: 42,
event_name: "$identify".to_string(),
set_properties: if set.is_null() {
vec![]
} else {
serde_json::to_vec(&set).unwrap()
},
set_once_properties: vec![],
unset_properties: vec![],
is_identified: identified,
last_seen_at: seen,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
// A last-seen-only advance is a change of its own: ingestion's
// direct write path persists it, so this path must too. The leader
// floors to the hour, so a raw mid-hour timestamp stores the hour.
let result = service
.update_person_properties(update(serde_json::Value::Null, None, Some(HOUR_MS + 1_234)))
.await
.expect("last-seen-only must be accepted")
.into_inner();
assert!(result.updated, "a last-seen-only advance earns a record");
let person = result.person.unwrap();
assert_eq!(person.version, 2);
assert_eq!(
person.last_seen_at,
Some(HOUR_MS),
"stored at hour granularity"
);
// A second event inside the stored hour is a no-op: the flooring
// bounds record volume at one per person-hour regardless of the
// caller's precision.
let result = service
.update_person_properties(update(
serde_json::Value::Null,
None,
Some(HOUR_MS + 900_000),
))
.await
.expect("a same-hour timestamp must be accepted")
.into_inner();
assert!(
!result.updated,
"a raw timestamp inside the stored hour must not produce a record"
);
// Identification is a change, and the newer last-seen merges in.
let result = service
.update_person_properties(update(
serde_json::Value::Null,
Some(true),
Some(5 * HOUR_MS),
))
.await
.expect("identification must apply")
.into_inner();
assert!(result.updated, "the false→true transition earns a record");
let person = result.person.unwrap();
assert_eq!(person.version, 3);
assert!(person.is_identified);
assert_eq!(person.last_seen_at, Some(5 * HOUR_MS));
// OR semantics: false does not revert, and re-asserting true is not a
// change.
let result = service
.update_person_properties(update(serde_json::Value::Null, Some(false), None))
.await
.expect("a false is a no-op, not an error")
.into_inner();
assert!(
!result.updated,
"identification never reverts through this RPC"
);
assert!(result.person.unwrap().is_identified);
let result = service
.update_person_properties(update(serde_json::Value::Null, Some(true), None))
.await
.expect("re-asserting true is accepted")
.into_inner();
assert!(
!result.updated,
"an already-identified person has nothing to change"
);
// A non-advancing last-seen is not a change: only forward movement
// of the stored value earns a record.
let result = service
.update_person_properties(update(serde_json::Value::Null, None, Some(3 * HOUR_MS)))
.await
.expect("a stale last-seen is a no-op, not an error")
.into_inner();
assert!(
!result.updated,
"a last-seen at or below the stored value must not produce a record"
);
// Max-merge: an older last-seen cannot lower the stored one, and a
// property change carries the newest value forward.
let result = service
.update_person_properties(update(
serde_json::json!({"plan": "pro"}),
None,
Some(3 * HOUR_MS),
))
.await
.expect("a property change with a stale last-seen must apply")
.into_inner();
assert!(result.updated);
let person = result.person.unwrap();
assert_eq!(person.version, 4);
assert_eq!(
person.last_seen_at,
Some(5 * HOUR_MS),
"a stale last-seen must not lower the stored value"
);
// A last-seen outside the writer's bindable range is discarded, not
// fatal: the update it rides on still lands.
let result = service
.update_person_properties(update(
serde_json::json!({"plan": "max"}),
None,
Some(i64::MAX),
))
.await
.expect("an absurd last-seen must not fail the update it rides on")
.into_inner();
assert!(result.updated);
let person = result.person.unwrap();
assert_eq!(person.version, 5);
assert_eq!(
person.last_seen_at,
Some(5 * HOUR_MS),
"an out-of-range last-seen must be discarded, not stored"
);
}