:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::sync::Arc;
use std::time::Instant;
use chrono::{TimeZone, Utc};
use dashmap::DashMap;
use metrics::{counter, histogram};
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeader;
use personhog_proto::personhog::types::v1::{
FencePersonRequest, FencePersonResponse, FencePersonsRequest, FencePersonsResponse,
FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest, GetPersonResponse,
Person, RaisePersonVersionRequest, RaisePersonVersionResponse, ReleaseFenceRequest,
ReleaseFenceResponse, ReleaseFencesRequest, ReleaseFencesResponse, SealedSourceSnapshot,
UpdatePersonPropertiesRequest, UpdatePersonPropertiesResponse,
};
use tokio::sync::{Mutex, OwnedMutexGuard};
use tonic::{Request, Response, Status};
use uuid::Uuid;
use personhog_common::partitioning::partition_for_person;
use personhog_coordination::authority::AuthorityClock;
use crate::cache::{
CacheLookup, CachedPerson, DirtyIndex, DirtyMark, PartitionedCache, PersonCacheKey, Stream,
};
use crate::coordination::ChangelogCaptures;
use crate::emitted::{EmittedVersionGuard, EmittedVersions};
use crate::fence::{
fenced_status, semantic_refusal, target_mark_status, FenceHealer, FenceMap, FenceState,
MarkVerifier,
};
#[cfg(test)]
use crate::fencing::FencedProducerConfig;
use crate::fencing::{FencedChangelogProducers, FencedProduceError};
use crate::inflight::{InflightGuard, InflightTracker};
use crate::person_update::{apply_property_updates, compute_event_property_updates};
use crate::pg::{load_person_from_pg, PgFallback};
use crate::recovery::ChangelogRecovery;
use crate::warnings::{SizeViolationWarning, WarningsProducer};
use personhog_common::properties::{
can_trim_property, jsonb_column_size, sanitize_for_jsonb, trim_properties_to_fit_size,
trim_properties_with_candidates, SanitizeStats, TrimResult,
};
mod lifecycle;
/// Mirrors the config's `fence_map_max_entries` default; production
/// overrides via [`PersonHogLeaderService::with_fence_capacity`].
const DEFAULT_FENCE_MAP_MAX_ENTRIES: usize = 250_000;
/// Admission-time property size limits, in `pg_column_size` (JSONB
/// binary) terms — the same units as the `check_properties_size`
/// constraint they exist to enforce.
#[derive(Clone, Copy)]
pub struct PropertySizeLimits {
/// Reject or trim above this (the constraint's ceiling).
pub threshold: usize,
/// Trim down to this, leaving headroom under the threshold.
pub trim_target: usize,
}
impl PropertySizeLimits {
/// Panics if `trim_target > threshold`: an inverted configuration would
/// let admission pass documents above the threshold untrimmed, silently
/// weakening the applyability guarantee. Refusing to construct makes the
/// admission path's `TrimResult::Fits` arm unreachable.
pub fn new(threshold: usize, trim_target: usize) -> Self {
assert!(
trim_target <= threshold,
"properties_trim_target ({trim_target}) must not exceed \
properties_size_threshold ({threshold})"
);
Self {
threshold,
trim_target,
}
}
}
/// Lands a document in the changelog and then the cache, as its own task,
/// so a handler dropped by its client cannot leave a record in flight
/// without the person lock or committed without the cache update.
struct Committer {
cache: Arc<PartitionedCache>,
dirty_index: Arc<DirtyIndex>,
emitted_versions: Arc<EmittedVersions>,
fenced: Arc<FencedChangelogProducers>,
/// Shared with the service; a mark names the partition's captured topic.
captures: ChangelogCaptures,
}
fn not_served(partition: u32) -> Status {
Status::failed_precondition(format!("partition {partition} is not served by this pod"))
}
/// A committed document, with the locks the handler took for it, so the
/// handler keeps them until it is done with the document.
struct Committed {
proto: Person,
_lock: OwnedMutexGuard<()>,
_inflight: InflightGuard,
}
/// Registers the revival outcomes at zero. A revival under a standing
/// death document is rare, so a series born at its first one would hide
/// that one from `increase()`.
fn register_revival_metrics() {
for outcome in [
"served",
"row_below_death",
"tombstoned",
"no_fallback",
"load_error",
] {
counter!("personhog_leader_revivals_total", "outcome" => outcome).increment(0);
}
drop(histogram!("personhog_leader_person_load_duration_ms", "source" => "revival"));
}
pub struct PersonHogLeaderService {
committer: Arc<Committer>,
cache: Arc<PartitionedCache>,
/// Per-key locks to serialize concurrent updates for the same person.
/// Prevents lost updates from concurrent get -> compute -> produce -> put
/// sequences, and thundering herd on PG fallback.
locks: Arc<DashMap<PersonCacheKey, Arc<Mutex<()>>>>,
/// Read-only PG fallback (pool + the table it reads) for cache miss.
fallback: Option<PgFallback>,
/// Per-partition inflight counter used to drive the handoff drain phase.
inflight: Arc<InflightTracker>,
/// What each served partition was acquired under; shared with the handler.
captures: ChangelogCaptures,
/// Persons whose latest acked state the writer may not have applied to
/// PG yet. Consulted on every cache miss: marked persons recover from
/// the changelog, unmarked persons' PG rows are known current.
dirty_index: Arc<DirtyIndex>,
recovery: Arc<ChangelogRecovery>,
size_limits: PropertySizeLimits,
warnings: WarningsProducer,
/// The in-process copy of the fence state (see the fence module for the
/// durability model: the marks in Postgres are the source of truth,
/// this map exists so the write path rejects without a database read).
/// Mutated only under the per-key lock.
fences: FenceMap,
/// Ghost-fence recovery, derived from the fallback pool at
/// construction. Absent without one (dev fixtures) — ghost fences
/// then last until the partition changes hands, as before.
fence_healer: Option<Arc<FenceHealer>>,
/// The committed-release mark check, one query per op per pod. Absent
/// without a fallback pool, in which case a release is refused.
mark_verifier: Option<MarkVerifier>,
/// Memory fuse for the fence map (see `fence_map_max_entries` in the
/// config for the full policy): at this many live fences, FencePerson
/// sheds new fences with RESOURCE_EXHAUSTED.
fence_map_max_entries: usize,
/// This pod's claim to serve, consulted before answering a strong read.
authority: Arc<AuthorityClock>,
/// Versions emitted without a confirmed outcome, so a later write for
/// the same person cannot reuse one.
emitted_versions: Arc<EmittedVersions>,
}
impl PersonHogLeaderService {
/// Refuse to answer as the partition's owner once this pod's lease
/// may have expired.
///
/// The cache is only authoritative while the lease behind it is, and
/// the keepalive's own detection cannot be relied on to notice: a
/// process that is stopped, starved, or wedged stops renewing and
/// stops noticing together, then keeps serving state the new owner
/// is already changing. Reading the published stamp here makes the
/// lapse self-enforcing — nothing has to be alive to apply it.
///
/// Refusal starts at the keepalive's renewal margin, strictly before
/// the coordinator could treat the lease as expired, so requests are
/// turned away while ownership is merely doubtful rather than after
/// it is wrong. `FailedPrecondition` is the admission fence's own
/// vocabulary: the router bounces and re-resolves toward whoever
/// actually owns the partition.
#[allow(clippy::result_large_err)]
fn check_authority(&self, partition: u32) -> Result<(), Status> {
let authority = &self.authority;
if authority.is_valid() {
return Ok(());
}
let reason = if authority.is_surrendered() {
"surrendered"
} else {
"stale"
};
counter!(
"personhog_leader_authority_lapsed_rejections_total",
"reason" => reason
)
.increment(1);
let since = authority.since_confirmed();
// Debug, not warn: the gate refuses *every* request for the whole
// duration of a lapse, so this fires at the pod's full read rate
// during exactly the incident someone would be reading logs for.
// The labelled counter above carries the rate and the cause.
tracing::debug!(
partition,
?since,
margin = ?authority.margin(),
"refusing to serve: no confirmed lease renewal within the margin"
);
Err(Status::failed_precondition(format!(
"serving authority lapsed: no confirmed lease renewal in {since:?}"
)))
}
/// The gate every ack passes: a valid lease and in-process ownership.
#[allow(clippy::result_large_err)]
fn assert_authoritative(&self, partition: u32) -> Result<(), Status> {
self.check_authority(partition)?;
self.validate_ownership(partition)
}
/// The fence RPCs' exit gate: a success answer re-proves authority at
/// the moment it is given. Only a stale Ok can wrongly settle a saga;
/// errors bounce and retry.
#[allow(clippy::result_large_err)]
fn authoritative_ok<T>(&self, partition: u32, resp: T) -> Result<Response<T>, Status> {
self.assert_authoritative(partition)?;
Ok(Response::new(resp))
}
#[allow(clippy::too_many_arguments)]
pub fn new(
cache: Arc<PartitionedCache>,
fallback: Option<PgFallback>,
locks: Arc<DashMap<PersonCacheKey, Arc<Mutex<()>>>>,
inflight: Arc<InflightTracker>,
captures: ChangelogCaptures,
dirty_index: Arc<DirtyIndex>,
recovery: Arc<ChangelogRecovery>,
size_limits: PropertySizeLimits,
warnings: WarningsProducer,
fences: FenceMap,
fenced: Arc<FencedChangelogProducers>,
authority: Arc<AuthorityClock>,
emitted_versions: Arc<EmittedVersions>,
) -> Self {
register_revival_metrics();
let committer = Arc::new(Committer {
cache: Arc::clone(&cache),
dirty_index: Arc::clone(&dirty_index),
emitted_versions: Arc::clone(&emitted_versions),
fenced,
captures: Arc::clone(&captures),
});
Self {
committer,
cache,
locks,
fence_healer: fallback
.as_ref()
.map(|f| Arc::new(FenceHealer::new(f.clone(), Arc::clone(&fences)))),
mark_verifier: fallback.as_ref().map(MarkVerifier::new),
fallback,
inflight,
captures,
dirty_index,
recovery,
size_limits,
warnings,
fences,
authority,
emitted_versions,
fence_map_max_entries: DEFAULT_FENCE_MAP_MAX_ENTRIES,
}
}
/// Override the fence-map capacity (`fence_map_max_entries` in the
/// config). Separate from `new` so the many test fixtures keep the
/// default without threading one more argument through.
pub fn with_fence_capacity(mut self, max_entries: usize) -> Self {
self.fence_map_max_entries = max_entries;
self
}
/// Verify the router's routing decision against the request body: the
/// partition a request arrived on must equal the partition derived from
/// the key it carries. A mismatch means a client stamped wrong
/// routing-key headers or the hash implementations diverged; serving it
/// would read or write through the wrong partition's cache, so fail
/// closed.
#[allow(clippy::result_large_err)]
fn validate_partition(
&self,
partition: u32,
team_id: i64,
person_id: i64,
) -> Result<(), Status> {
let modulus = self
.captures
.get(&partition)
.map(|c| c.config.partitions)
.ok_or_else(|| not_served(partition))?;
let expected = partition_for_person(team_id, person_id, modulus);
if partition != expected {
counter!("personhog_leader_partition_mismatch_total").increment(1);
// A precondition, so the router bounces and retries, landing once it adopts the new count.
return Err(Status::failed_precondition(format!(
"x-partition {partition} does not match partition {expected} \
derived from team_id={team_id} person_id={person_id}"
)));
}
Ok(())
}
/// The fence RPCs must never succeed on a pod that does not serve the
/// partition. A misrouted release that removed nothing and returned
/// OK would leave the real owner's fence in place while the saga
/// believes it released — a person frozen with no retry coming. The
/// handoff guard is not enough on its own: `release_partition`
/// unfences, so a pod that has already handed the partition off looks
/// unfenced. Rejecting sends the saga's retry to the current owner.
#[allow(clippy::result_large_err)]
fn validate_ownership(&self, partition: u32) -> Result<(), Status> {
if !self.cache.has_partition(partition) {
return Err(not_served(partition));
}
Ok(())
}
fn record_cache_hit() {
counter!(
"personhog_leader_person_loads_total",
"source" => "cache", "outcome" => "ok"
)
.increment(1);
}
/// Recover a cache miss from the right source. A person in the dirty
/// index has acked state the writer may not have applied to PG yet, so
/// the PG row cannot be trusted — recover the full latest state from
/// the changelog record at the marked offset instead. If that fetch
/// fails, the only honest answer is a retryable error: falling back to
/// PG would serve exactly the staleness this index exists to prevent.
/// Unmarked persons' PG rows are known current — but only while this
/// pod owns the partition, so the no-mark path re-checks ownership
/// before trusting PG. Assumes the caller holds the per-key lock.
async fn recover_or_load(
&self,
partition: u32,
key: &PersonCacheKey,
) -> Result<CachedPerson, Status> {
let Some(mark) = self.dirty_index.get(key) else {
// "No mark" means PG is current — but only while this pod owns
// the partition. Handoffs drain writes, not reads, so a read
// admitted before the freeze can still be executing here when
// `release_partition` clears the partition's marks (the new
// owner rebuilds its own), and to that reader a still-dirty
// person now looks safe to load from PG. Re-checking the cache
// AFTER the index read settles which world we're in: release
// drops the cache partition first and clears marks second, so
// if this miss was caused by release, the partition is already
// gone from the cache and we fail closed; if the partition is
// still present, the marks were intact when we read them and
// the absence is genuine. There is no interleaving that gets
// past both checks into a stale PG read.
return match self.cache.get(partition, key) {
CacheLookup::Found(person) => {
Self::record_cache_hit();
Ok(person)
}
CacheLookup::PersonNotFound => self.load_from_pg(partition, key).await,
CacheLookup::PartitionNotOwned => Err(Status::failed_precondition(format!(
"partition {partition} not owned by this leader"
))),
};
};
let started = Instant::now();
let result = self.recovery.fetch_person_at(&mark, key).await;
histogram!("personhog_leader_person_load_duration_ms", "source" => "changelog")
.record(started.elapsed().as_secs_f64() * 1000.0);
match result {
Ok(person) => {
counter!(
"personhog_leader_person_loads_total",
"source" => "changelog", "outcome" => "ok"
)
.increment(1);
self.cache.put(partition, key.clone(), person.clone());
Ok(person)
}
Err(e) => {
counter!(
"personhog_leader_person_loads_total",
"source" => "changelog", "outcome" => "error"
)
.increment(1);
tracing::error!(
team_id = key.team_id,
person_id = key.person_id,
offset = mark.offset,
error = %e,
"changelog recovery failed for dirty person"
);
Err(Status::unavailable(
"person state is pending durable write and changelog recovery failed; retry",
))
}
}
}
/// Load a person from PG and populate the cache. Assumes the caller
/// holds the per-key lock.
async fn load_from_pg(
&self,
partition: u32,
key: &PersonCacheKey,
) -> Result<CachedPerson, Status> {
let Some(fallback) = &self.fallback else {
// Without the pool a cache miss answers NotFound, which callers
// read as authoritative death; production always sets it.
return Err(Status::not_found(format!(
"person not found: team_id={}, person_id={}",
key.team_id, key.person_id
)));
};
let started = Instant::now();
let result = load_person_from_pg(&fallback.pool, &fallback.table, key).await;
histogram!("personhog_leader_person_load_duration_ms", "source" => "pg")
.record(started.elapsed().as_secs_f64() * 1000.0);
match result {
Ok(Some(person)) => {
counter!(
"personhog_leader_person_loads_total",
"source" => "pg", "outcome" => "ok"
)
.increment(1);
self.cache.put(partition, key.clone(), person.clone());
Ok(person)
}
Ok(None) => {
counter!(
"personhog_leader_person_loads_total",
"source" => "pg", "outcome" => "not_found"
)
.increment(1);
Err(Status::not_found(format!(
"person not found: team_id={}, person_id={}",
key.team_id, key.person_id
)))
}
Err(e) => {
counter!(
"personhog_leader_person_loads_total",
"source" => "pg", "outcome" => "error"
)
.increment(1);
counter!("personhog_leader_pg_fallback_errors_total").increment(1);
tracing::error!(
team_id = key.team_id,
person_id = key.person_id,
error = %e,
"PG fallback query failed"
);
Err(Status::internal("failed to load person from database"))
}
}
}
/// A live PG row above the death version can only be a stub-create
/// revival under the same id. Caller holds the per-key lock.
async fn serve_revival(
&self,
partition: u32,
key: &PersonCacheKey,
person: CachedPerson,
) -> Result<CachedPerson, Status> {
if !person.is_deleted {
return Ok(person);
}
let Some(fallback) = &self.fallback else {
counter!("personhog_leader_revivals_total", "outcome" => "no_fallback").increment(1);
return Ok(person);
};
let started = Instant::now();
let result = load_person_from_pg(&fallback.pool, &fallback.table, key).await;
histogram!("personhog_leader_person_load_duration_ms", "source" => "revival")
.record(started.elapsed().as_secs_f64() * 1000.0);
match result {
Ok(Some(row)) if row.version > person.version => {
counter!("personhog_leader_revivals_total", "outcome" => "served").increment(1);
tracing::info!(
team_id = key.team_id,
person_id = key.person_id,
death_version = person.version,
version = row.version,
"serving a person revived in Postgres over its death document"
);
self.cache.put(partition, key.clone(), row.clone());
Ok(row)
}
Ok(Some(row)) => {
// The tombstone precedes the release, so a live row this
// low is a pre-death read; it must not revive the person.
counter!("personhog_leader_revivals_total", "outcome" => "row_below_death")
.increment(1);
tracing::warn!(
team_id = key.team_id,
person_id = key.person_id,
death_version = person.version,
version = row.version,
"a live Postgres row sits at or below the death document"
);
Ok(person)
}
Ok(None) => {
counter!("personhog_leader_revivals_total", "outcome" => "tombstoned").increment(1);
Ok(person)
}
Err(e) => {
counter!("personhog_leader_revivals_total", "outcome" => "load_error").increment(1);
counter!("personhog_leader_pg_fallback_errors_total").increment(1);
tracing::error!(
team_id = key.team_id,
person_id = key.person_id,
error = %e,
"PG revival check failed"
);
// Callers read not-found as gone, which this could not establish.
Err(Status::unavailable(
"could not check whether a destroyed person was revived; retry",
))
}
}
}
/// Look up a person from cache, falling back to PG on miss.
/// Acquires a per-key lock.
async fn lookup_or_load(
&self,
partition: u32,
key: &PersonCacheKey,
) -> Result<CachedPerson, Status> {
// Fast path: cache hit (no lock needed); a death document takes the locked path.
match self.cache.get(partition, key) {
CacheLookup::Found(person) if !person.is_deleted => {
Self::record_cache_hit();
return Ok(person);
}
CacheLookup::Found(_) => {}
CacheLookup::PartitionNotOwned => {
return Err(Status::failed_precondition(format!(
"partition {} not owned by this leader",
partition
)));
}
CacheLookup::PersonNotFound => {}
}
// Cache miss -- acquire per-key lock to prevent thundering herd
let mutex = self.locks.entry(key.clone()).or_default().value().clone();
let _guard = mutex.lock().await;
// Double-check cache -- another request may have loaded it
let person = if let CacheLookup::Found(person) = self.cache.get(partition, key) {
Self::record_cache_hit();
person
} else {
self.recover_or_load(partition, key).await?
};
self.serve_revival(partition, key, person).await
}
/// Look up a person from cache, falling back to PG on miss.
/// The caller must already hold the per-key lock.
async fn lookup_or_load_locked(
&self,
partition: u32,
key: &PersonCacheKey,
) -> Result<CachedPerson, Status> {
let person = match self.cache.get(partition, key) {
CacheLookup::Found(person) => {
Self::record_cache_hit();
person
}
CacheLookup::PartitionNotOwned => {
return Err(Status::failed_precondition(format!(
"partition {} not owned by this leader",
partition
)))
}
CacheLookup::PersonNotFound => self.recover_or_load(partition, key).await?,
};
self.serve_revival(partition, key, person).await
}
/// The shared tail of every document write: refuse unapplyable records,
/// produce to Kafka first, then dirty-mark and update the cache, so
/// readers only ever see durably committed state. The person lock and
/// the in-flight slot travel with the commit and come back with the
/// document.
async fn commit_document(
&self,
partition: u32,
cache_key: &PersonCacheKey,
person: CachedPerson,
lock: OwnedMutexGuard<()>,
inflight: InflightGuard,
) -> Result<Committed, Status> {
// A record the writer cannot bind must never reach the changelog —
// no consumer downstream can apply or repair it.
if let Err(reason) = assert_writeable(&person) {
counter!("personhog_leader_unwriteable_state_total").increment(1);
tracing::error!(
team_id = cache_key.team_id,
person_id = cache_key.person_id,
reason,
"refusing to produce an unapplyable changelog record"
);
return Err(Status::internal(format!(
"person state is not writeable: {reason}"
)));
}
let proto = cached_person_into_proto(person.clone());
// Re-check before producing, for the same reason the read path
// re-checks before answering: admission proves nothing about the
// moment the record lands. Between the check at entry and here a
// write can wait on the per-key lock behind another produce, and
// on a changelog recovery — long enough for a starved keepalive's
// stamp to age out, for the lease to expire at etcd, and for the
// coordinator to warm a successor past the point this record
// would land.
self.check_authority(partition)?;
// From here the record may reach the changelog whatever happens
// to this request, including the request ceasing to exist when
// the client's deadline expires, so the rest runs as its own
// task: it finishes under the person lock, lands the record in
// the cache, and only then lets the lock go.
let committer = Arc::clone(&self.committer);
let (team_id, person_id) = (cache_key.team_id, cache_key.person_id);
let cache_key = cache_key.clone();
tokio::spawn(async move {
committer
.commit(partition, cache_key, person, proto, lock, inflight)
.await
})
.await
.unwrap_or_else(|e| {
// The floor stays raised through the unwind, so the version
// is spent; what is lost is only the outcome.
tracing::error!(
team_id,
person_id,
partition,
error = %e,
"commit task ended without reporting an outcome"
);
Err(Status::internal(format!("commit task: {e}")))
})
}
}
impl Committer {
/// The producer was acquired on the capture's topic, so its records land there.
#[allow(clippy::result_large_err)]
fn stream_of(&self, partition: u32) -> Result<Stream, Status> {
let (topic, cluster) = self
.captures
.get(&partition)
.map(|c| (Arc::clone(&c.topic), c.cluster.clone()))
.ok_or_else(|| not_served(partition))?;
Ok(Stream {
topic,
cluster,
partition,
})
}
async fn commit(
&self,
partition: u32,
cache_key: PersonCacheKey,
person: CachedPerson,
proto: Person,
lock: OwnedMutexGuard<()>,
inflight: InflightGuard,
) -> Result<Committed, Status> {
let stream = self.stream_of(partition)?;
// The guard is what makes the version un-reusable if this task
// never reports.
let mut emitted = EmittedVersionGuard::new(
Arc::clone(&self.emitted_versions),
partition,
cache_key.clone(),
person.version,
);
emitted.emitting();
// Produce to Kafka first, then update the cache on success.
// Readers only ever see durably committed state.
let offset = match self.fenced.produce(partition, &proto).await {
Ok(offset) => offset,
// The broker fenced this pod: a newer owner holds the
// partition, so this claim is stale. FailedPrecondition
// is the admission fence's own vocabulary — the router
// classifies it as a bounce and re-resolves toward the
// real owner.
Err(e @ FencedProduceError::Fenced) => {
// Rejected at the broker, so the record does not
// exist and its version is free.
emitted.discarded();
tracing::error!(
team_id = cache_key.team_id,
person_id = cache_key.person_id,
partition,
"changelog producer fenced; rejecting write as stale owner"
);
// No local reaction beyond failing the write: a fence
// proves someone newer took the id, not that this pod
// lost the partition. `heal_fence` re-takes the epoch
// on the next convergence, once the stamp allows it.
return Err(Status::failed_precondition(format!(
"partition ownership fenced: {e}"
)));
}
// The partition moved and this window's outcome never
// came back, so the router gets its ownership answer but
// the version cannot be handed back.
Err(e @ FencedProduceError::FencedUncertain(_)) => {
// The record may or may not be in the log, so the
// version stays spent.
counter!(
"personhog_leader_indeterminate_outcomes_total",
"fenced" => "true"
)
.increment(1);
tracing::error!(
team_id = cache_key.team_id,
person_id = cache_key.person_id,
partition,
error = %e,
"changelog producer fenced with an unknown outcome; version kept spent"
);
return Err(Status::failed_precondition(format!(
"partition ownership fenced: {e}"
)));
}
Err(e @ FencedProduceError::NotAcquired) => {
emitted.discarded();
tracing::warn!(
team_id = cache_key.team_id,
person_id = cache_key.person_id,
partition,
"no changelog fence held for partition; rejecting write"
);
return Err(Status::failed_precondition(format!(
"partition fence not held: {e}"
)));
}
// Whether the record became visible is unknown, so the
// version stays spent: a retry reusing it would race a
// record that may already have committed.
Err(e @ FencedProduceError::Indeterminate(_)) => {
// The retry derives past the spent version.
counter!(
"personhog_leader_indeterminate_outcomes_total",
"fenced" => "false"
)
.increment(1);
tracing::error!(
team_id = cache_key.team_id,
person_id = cache_key.person_id,
partition,
error = %e,
"changelog commit outcome unknown; version kept spent"
);
return Err(Status::unknown(format!(
"person state may or may not have been stored: {e}"
)));
}
// The window aborted, so nothing became visible and
// ABORTED is the code clients retry on.
Err(e) => {
// Nothing became visible, so the version is free.
emitted.discarded();
tracing::error!(
team_id = cache_key.team_id,
person_id = cache_key.person_id,
error = %e,
"failed to produce person state changelog"
);
return Err(Status::aborted(format!(
"failed to durably store person state: {e}"
)));
}
};
// Mark before the cache insert: a reader that misses the cache in
// the gap sees the mark and recovers this exact record from the
// changelog. The mark outlives eviction and is pruned once the
// writer's committed offset passes it.
self.dirty_index.mark(
cache_key.clone(),
DirtyMark {
version: person.version,
offset,
partition,
stream,
is_deleted: person.is_deleted,
},
);
self.cache.put(partition, cache_key, person);
// The cache carries the version now, so the floor has nothing
// left to say.
emitted.resolved();
Ok(Committed {
proto,
_lock: lock,
_inflight: inflight,
})
}
}
impl PersonHogLeaderService {
/// The write path's fence conditional: an in-memory lookup and nothing
/// else. The map is authoritative here — a fence that outlives its op
/// is not the leader's to detect, because the op being unfinished is
/// exactly what a live mark means (see the fence module's ownership
/// model). Returns the fence to reject with, or None when the person
/// is writable. Assumes the caller holds the per-key lock.
fn check_fence(&self, cache_key: &PersonCacheKey) -> Option<FenceState> {
self.fences.get(cache_key).map(|entry| *entry.value())
}
}
/// The first instant Postgres accepts in a `timestamptz`: Julian day 0,
/// which Postgres documents as 4713 BC (Julian calendar) and chrono
/// renders as -4713-11-24 (proleptic Gregorian). chrono reaches much
/// further back, so this is the binding bound.
const PG_TIMESTAMPTZ_MIN_MS: i64 = -210_866_803_200_000;
/// Upper bound on the epoch-millisecond timestamps the leader will put in
/// a changelog record: the last instant of year 9999. Inside both chrono's
/// and Postgres's ranges; nothing legitimate sits above it.
const MAX_EPOCH_MS_YEAR_9999: i64 = 253_402_300_799_999;
/// Whether the writer can bind this epoch-millisecond value as a
/// `timestamptz`: chrono must represent it and Postgres must accept it.
/// Rows written before personhog carry a client's zero `DateTime` (year 1)
/// as `created_at`, and those must keep writing.
fn writeable_epoch_ms(ms: i64) -> bool {
(PG_TIMESTAMPTZ_MIN_MS..=MAX_EPOCH_MS_YEAR_9999).contains(&ms)
&& Utc.timestamp_millis_opt(ms).single().is_some()
}
/// The granularity `last_seen_at` is stored at. Matches ingestion's own
/// hourly truncation so both write paths produce identical stored values.
const MS_PER_HOUR: i64 = 3_600_000;
/// Floors to the hour on either side of the epoch; `%` would round a
/// negative value up to the next hour instead.
fn floor_to_hour(ms: i64) -> i64 {
ms.div_euclid(MS_PER_HOUR) * MS_PER_HOUR
}
/// The changelog contract: every produced record must be applyable by the
/// writer's upsert verbatim. Properties are guaranteed by admission
/// (NUL sanitization plus the exact size measure); this checks the
/// remaining writer-bound fields against the writer's bind conversions —
/// uuid must parse, team_id must fit the column's `integer`, and the
/// timestamps must convert to a `timestamptz`. The legacy jsonb columns
/// have no cache field and are unconditionally empty in
/// `cached_person_into_proto`, so a record structurally cannot carry
/// values the writer would refuse there.
fn assert_writeable(p: &CachedPerson) -> Result<(), String> {
if Uuid::parse_str(&p.uuid).is_err() {
return Err("uuid does not parse as a UUID".to_string());
}
if p.team_id <= 0 || p.team_id > i32::MAX as i64 {
return Err(format!(
"team_id {} is outside the column's integer range",
p.team_id
));
}
if !writeable_epoch_ms(p.created_at) {
return Err(format!(
"created_at epoch {} is not representable as a timestamp",
p.created_at
));
}
if let Some(t) = p.last_seen_at {
if !writeable_epoch_ms(t) {
return Err(format!(
"last_seen_at epoch {t} is not representable as a timestamp"
));
}
}
Ok(())
}
fn cached_person_into_proto(p: CachedPerson) -> Person {
Person {
id: p.id,
uuid: p.uuid,
team_id: p.team_id,
properties: p.properties,
properties_last_updated_at: Vec::new(),
properties_last_operation: Vec::new(),
created_at: p.created_at,
version: p.version,
is_identified: p.is_identified,
is_user_id: None,
last_seen_at: p.last_seen_at,
is_deleted: p.is_deleted,
}
}
/// Parse a JSON-map wire field (empty bytes mean an empty map), refusing
/// anything that is not a JSON object.
// See `partition_from_metadata` for why `result_large_err` is allowed.
#[allow(clippy::result_large_err)]
fn parse_json_object_field(bytes: &[u8], field: &str) -> Result<serde_json::Value, Status> {
if bytes.is_empty() {
return Ok(serde_json::Value::Object(serde_json::Map::new()));
}
let value: serde_json::Value = serde_json::from_slice(bytes)
.map_err(|e| Status::invalid_argument(format!("invalid {field} JSON: {e}")))?;
if !value.is_object() {
return Err(Status::invalid_argument(format!(
"{field} must be a JSON object"
)));
}
Ok(value)
}
/// A sealed source snapshot after admission: identity-checked against the
/// fold's target, JSON-parsed, and sanitized.
#[derive(Debug)]
struct SealedSnapshot {
ordinal: i32,
properties: serde_json::Value,
version: i64,
created_at: i64,
last_seen_at: Option<i64>,
}
/// Admit the fold request's sealed snapshots: validate, parse, and
/// sanitize each, returning them in ordinal order plus what sanitization
/// rewrote. Every refusal is a deterministic `InvalidArgument`: each
/// snapshot must be a plausible living source of the target — same team,
/// a different person, not a death document — and ordinals must be
/// unique, or precedence would be ambiguous. A mismatch can only be a
/// saga bug, and folding it silently would launder it into the target.
// See `partition_from_metadata` for why `result_large_err` is allowed.
#[allow(clippy::result_large_err)]
fn parse_sealed_snapshots(
sealed_snapshots: &[SealedSourceSnapshot],
team_id: i64,
person_id: i64,
) -> Result<(Vec<SealedSnapshot>, SanitizeStats), Status> {
let mut stats = SanitizeStats::default();
let mut seen_ordinals = std::collections::HashSet::with_capacity(sealed_snapshots.len());
let mut snapshots: Vec<SealedSnapshot> = Vec::with_capacity(sealed_snapshots.len());
for snapshot in sealed_snapshots {
let Some(person) = &snapshot.person else {
return Err(Status::invalid_argument(
"sealed snapshot is missing its person",
));
};
if person.team_id != team_id {
return Err(Status::invalid_argument(
"sealed snapshot belongs to a different team than the target",
));
}
if person.id == person_id {
return Err(Status::invalid_argument(
"sealed snapshot is the merge target itself",
));
}
if person.is_deleted {
return Err(Status::invalid_argument(
"sealed snapshot is a death document; only living sealed state folds",
));
}
if !seen_ordinals.insert(snapshot.ordinal) {
return Err(Status::invalid_argument(
"sealed snapshots carry a duplicate ordinal; precedence would be ambiguous",
));
}
let mut properties =
parse_json_object_field(&person.properties, "sealed snapshot properties")?;
let snapshot_stats = sanitize_for_jsonb(&mut properties);
stats.nul_strings += snapshot_stats.nul_strings;
stats.clamped_numbers += snapshot_stats.clamped_numbers;
snapshots.push(SealedSnapshot {
ordinal: snapshot.ordinal,
properties,
version: person.version,
created_at: person.created_at,
last_seen_at: person.last_seen_at,
});
}
// Precedence comes from the recorded pair order, not from the
// request: a re-drive that lists sources differently must fold the
// same document.
snapshots.sort_by_key(|snapshot| snapshot.ordinal);
Ok((snapshots, stats))
}
/// Extract the routing partition from the `x-partition` request-metadata
/// header. The router stamps this on every leader call after hashing
/// `(team_id, person_id)`; its absence means a misrouted or malformed
/// request, so we fail closed with `InvalidArgument` rather than guessing.
// `Status` is the idiomatic tonic error throughout this service; the small
// `Ok(u32)` against a large `Status` trips `result_large_err`, but boxing
// here would diverge from every other handler's signature.
#[allow(clippy::result_large_err)]
fn partition_from_metadata<T>(request: &Request<T>) -> Result<u32, Status> {
request
.metadata()
.get("x-partition")
.ok_or_else(|| Status::invalid_argument("missing x-partition metadata"))?
.to_str()
.map_err(|_| Status::invalid_argument("x-partition metadata is not valid ASCII"))?
.parse::<u32>()
.map_err(|_| Status::invalid_argument("x-partition metadata is not a valid u32"))
}
#[tonic::async_trait]
impl PersonHogLeader for PersonHogLeaderService {
async fn get_person(
&self,
request: Request<GetPersonRequest>,
) -> Result<Response<GetPersonResponse>, Status> {
let partition = partition_from_metadata(&request)?;
let req = request.into_inner();
self.validate_partition(partition, req.team_id, req.person_id)?;
self.check_authority(partition)?;
let cache_key = PersonCacheKey {
team_id: req.team_id,
person_id: req.person_id,
};
let person = self.lookup_or_load(partition, &cache_key).await?;
// Re-check before answering. The load can wait — on the per-key
// lock behind another request's produce, or on a changelog
// recovery — for long enough that a claim valid at admission has
// lapsed by the time there is something to return, and it is the
// answer, not the arrival, that has to be backed by ownership.
self.check_authority(partition)?;
// A recovered death document is an authoritative not-found: the
// person was destroyed and this entry closes its stream.
if person.is_deleted {
return Err(Status::not_found("person is destroyed"));
}
Ok(Response::new(GetPersonResponse {
person: Some(cached_person_into_proto(person)),
}))
}
async fn update_person_properties(
&self,
request: Request<UpdatePersonPropertiesRequest>,
) -> Result<Response<UpdatePersonPropertiesResponse>, Status> {
let partition = partition_from_metadata(&request)?;
let req = request.into_inner();
self.validate_partition(partition, req.team_id, req.person_id)?;
// A write is serving too. Broker-enforced fencing covers this
// path when it is on, but it cannot be turned on first — startup
// refuses fencing without the gate — so every fleet passes
// through a window where the gate is the only thing standing
// between a pod that stopped renewing and a write the successor
// will never see. The lease-loss path surrenders before it
// drains, deliberately, and until this check existed only reads
// honoured that: writes stayed admitted until the local fence
// landed, behind a watch teardown and a task join.
self.check_authority(partition)?;
// Admit the write as inflight, unless the partition is fenced. A
// fenced partition has drained for handoff: every router acked the
// freeze, so this write can only come from a router with a stale
// view — accepting it would produce past the Kafka HWM that the new
// owner's warming snapshots, silently losing the write. Admission
// and the fence check are one atomic operation (`try_begin`): the
// inflight increment precedes the check, so the drain either waits
// for this write or this write sees the fence. Reads are unaffected
// — the frozen state stays the latest until cutover. The handoff
// protocol waits for the per-partition inflight count to drop to
// zero before advancing; combined with sync-acked produces, a zero
// count implies every acked write is durable in Kafka. The guard
// moves into the commit, which holds it through the outcome.
let Some(inflight_guard) = self.inflight.try_begin(partition) else {
return Err(Status::failed_precondition(format!(
"partition {partition} is fenced for handoff; writes are rejected"
)));
};
let cache_key = PersonCacheKey {
team_id: req.team_id,
person_id: req.person_id,
};
// Parse JSON before acquiring the per-key lock to minimize lock hold time
let set_properties: serde_json::Value = if req.set_properties.is_empty() {
serde_json::Value::Object(serde_json::Map::new())
} else {
serde_json::from_slice(&req.set_properties).map_err(|e| {
Status::invalid_argument(format!("invalid set_properties JSON: {e}"))
})?
};
let set_once_properties: serde_json::Value = if req.set_once_properties.is_empty() {
serde_json::Value::Object(serde_json::Map::new())
} else {
serde_json::from_slice(&req.set_once_properties).map_err(|e| {
Status::invalid_argument(format!("invalid set_once_properties JSON: {e}"))
})?
};
// Sanitize the request values before diffing so no-op detection
// compares in sanitized space: cached state holds the sanitized
// form, and an unsanitized repeat of the same value must hit the
// no-change fast path instead of producing a fresh record every
// time. The post-merge sanitize below stays as the admission
// guarantee (it also covers legacy dirt already in the cache).
let mut set_properties = set_properties;
let mut set_once_properties = set_once_properties;
let input_stats = {
let mut stats = sanitize_for_jsonb(&mut set_properties);
let once = sanitize_for_jsonb(&mut set_once_properties);
stats.nul_strings += once.nul_strings;
stats.clamped_numbers += once.clamped_numbers;
stats
};
if input_stats.nul_strings > 0 {
counter!("personhog_leader_properties_nul_sanitized_total")
.increment(input_stats.nul_strings);
}
if input_stats.clamped_numbers > 0 {
counter!("personhog_leader_properties_numbers_clamped_total")
.increment(input_stats.clamped_numbers);
}
// Unset targets must match keys as stored, i.e. sanitized.
let unset_properties: Vec<String> = req
.unset_properties
.iter()
.map(|k| k.replace('\u{0000}', "\u{FFFD}"))
.collect();
// Per-key lock serializes concurrent updates for the same person.
// The wait is measured because it is the queueing component of
// handler latency: with every acked write holding the lock
// through its acks=all produce, per-person throughput is capped
// near 1/produce-latency, and contending updates spend their
// time here — invisible in the produce histogram.
let mutex = self
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let lock_wait = std::time::Instant::now();
let guard = mutex.lock_owned().await;
histogram!("personhog_leader_person_lock_wait_ms")
.record(lock_wait.elapsed().as_secs_f64() * 1000.0);
// Admission check before any work: if the dirty index is at
// capacity and this person is not already marked, acking the write
// would leave it durable but unmarked — reopening the
// stale-fallback hole on eviction. Shed instead; the index drains
// (and admission resumes) as the writer catches up.
if !self.dirty_index.can_admit(&cache_key) {
counter!("personhog_leader_writes_shed_total", "reason" => "dirty_index_full")
.increment(1);
return Err(Status::resource_exhausted(
"dirty index at capacity: the writer is behind and this person's write cannot \
be tracked; retry later",
));
}
// A person fenced by a live lifecycle op rejects writes until the
// fence is released; reads are unaffected. Checked under the
// per-key lock so a concurrent FencePerson cannot be missed.
if let Some(fence) = self.check_fence(&cache_key) {
counter!("personhog_leader_writes_fenced_total").increment(1);
// Off the request path: if this fence is a ghost (its op
// already settled), the healer drops it and a retry goes
// through.
if let Some(healer) = &self.fence_healer {
healer.maybe_heal(cache_key.clone(), fence);
}
return Err(fenced_status(&fence));
}
let person = self.lookup_or_load_locked(partition, &cache_key).await?;
// The load can park past the claim; the no-change answers below
// carry cached state, so they need ownership like any read.
self.check_authority(partition)?;
// A destroyed person answers not-found — the caller re-resolves;
// post-death the distinct id may have been reborn as a new person.
if person.is_deleted {
return Err(Status::not_found("person is destroyed"));
}
// One parse per update: the cache stores properties serialized,
// and this handler reads them as a map throughout.
let person_properties = person
.parse_properties()
.map_err(|e| Status::internal(format!("cached properties unparseable: {e}")))?;
// Compute property updates
let updates = compute_event_property_updates(
&req.event_name,
&set_properties,
&set_once_properties,
&unset_properties,
&person_properties,
req.force_update,
);
// OR-merge: identification never reverts through this RPC, so
// only a true that finds false is a change. It counts as one on
// its own — an $identify with no property diffs must still
// produce a record.
let identified_now = person.is_identified || req.is_identified == Some(true);
let identity_changed = identified_now != person.is_identified;
// last_seen_at is request-borne and best-effort: an out-of-range
// value is discarded rather than failing the update, so the
// acked ⇒ writeable invariant never hinges on it. In-range values
// are floored to the hour — Node's startOf('hour') on its
// UTC-normalized timestamps, expressed in epoch terms — so record
// volume is bounded at one per person-hour by construction, not
// by trusting callers to coarsen.
let requested_last_seen = req.last_seen_at.filter(|t| writeable_epoch_ms(*t));
if requested_last_seen != req.last_seen_at {
counter!("personhog_leader_last_seen_discarded_total").increment(1);
}
let requested_last_seen = requested_last_seen.map(floor_to_hour);
// Max-merge: the stored value only ever advances, and an advance
// is a change in its own right — ingestion's direct write path
// persists a last-seen-only advance, so this path must too.
let merged_last_seen = person.last_seen_at.max(requested_last_seen);
let last_seen_changed = merged_last_seen != person.last_seen_at;
// Fast path: no diffs detected, skip the clone in apply_property_updates
if !updates.has_changes && !identity_changed && !last_seen_changed {
counter!("personhog_leader_updates_total", "outcome" => "no_change").increment(1);
return Ok(Response::new(UpdatePersonPropertiesResponse {
person: Some(cached_person_into_proto(person)),
updated: false,
}));
}
// Filtered-only changes are answered without writing, matching the
// Postgres suppression; a scalar move or force promotes everything.
if !updates.has_non_filtered_changes && !identity_changed && !last_seen_changed {
counter!("personhog_leader_updates_total", "outcome" => "filtered_only").increment(1);
return Ok(Response::new(UpdatePersonPropertiesResponse {
person: Some(cached_person_into_proto(person)),
updated: false,
}));
}
// Slow path: apply diffs and check if the values actually changed
// (has_changes can be true when $set sends the same value that already exists)
let (new_properties, actually_updated) =
apply_property_updates(&updates, &person_properties);
if !actually_updated && !identity_changed && !last_seen_changed {
counter!("personhog_leader_updates_total", "outcome" => "no_change").increment(1);
return Ok(Response::new(UpdatePersonPropertiesResponse {
person: Some(cached_person_into_proto(person)),
updated: false,
}));
}
// Admission-time size enforcement, hoisted from the writer: every
// acked record must be applyable by the writer verbatim, or the
// cache and changelog would carry state Postgres never gets (the
// stale-serve hole the dirty index exists to close). The measure
// is the constraint's own — pg_column_size of the JSONB encoding
// — so admitted rows cannot violate it at apply time.
let mut new_properties = new_properties;
// Rewrite the merged state into jsonb-safe form before measuring:
// NUL sanitization (Node-pipeline parity — Postgres refuses it)
// and extreme-float clamping (Postgres's expanded numeric
// rendering would otherwise be unparseable on the way back). The
// measured size is then the stored size.
let sanitize_stats = sanitize_for_jsonb(&mut new_properties);
if sanitize_stats.nul_strings > 0 {
counter!("personhog_leader_properties_nul_sanitized_total")
.increment(sanitize_stats.nul_strings);
}
if sanitize_stats.clamped_numbers > 0 {
counter!("personhog_leader_properties_numbers_clamped_total")
.increment(sanitize_stats.clamped_numbers);
}
let jsonb_size = jsonb_column_size(&new_properties);
if jsonb_size > self.size_limits.threshold {
// Policy mirror of the Node pipeline's
// `handleOversizedPersonProperties`: trimming is remediation
// for rows already oversized in storage (they predate the
// constraint, or another writer produced them) — the stored
// properties are trimmed to the target and the triggering
// update's property changes are discarded, exactly as Node
// retries with trimmed existing state. An update that would
// newly push a within-limit row over the ceiling is rejected
// outright: a rejection with a warning is a deliberate,
// visible outcome where a silent trim would be arbitrary
// deferred data loss. Warnings and errors carry sizes, never
// property values.
let existing_size = jsonb_column_size(&person_properties);
if existing_size >= self.size_limits.threshold {
match trim_properties_to_fit_size(&person_properties, self.size_limits.trim_target)
{
TrimResult::Trimmed(trimmed) => {
counter!("personhog_leader_properties_trimmed_total").increment(1);
self.warnings.emit(&SizeViolationWarning {
team_id: cache_key.team_id,
person_uuid: person.uuid.clone(),
message: "Oversized person properties were trimmed to fit the size \
limit; the update that surfaced them was discarded"
.to_string(),
});
new_properties = trimmed;
}
// Reachable only when trim_target == threshold and the
// stored size sits exactly on it: nothing to trim, but
// the update still cannot apply — keep the stored
// state, discarding the update like the arm above.
TrimResult::Fits => {
new_properties = person_properties.clone();
}
TrimResult::CannotFit => {
counter!(
"personhog_leader_updates_total",
"outcome" => "rejected_unremediable"
)
.increment(1);
self.warnings.emit(&SizeViolationWarning {
team_id: cache_key.team_id,
person_uuid: person.uuid.clone(),
message: "Person properties exceed the size limit and could not be \
trimmed; the update was rejected"
.to_string(),
});
return Err(Status::invalid_argument(format!(
"person properties exceed the size limit: stored state is \
{existing_size} bytes (jsonb) and protected properties alone exceed \
the {} trim target",
self.size_limits.trim_target,
)));
}
}
} else {
counter!("personhog_leader_updates_total", "outcome" => "rejected_oversized")
.increment(1);
self.warnings.emit(&SizeViolationWarning {
team_id: cache_key.team_id,
person_uuid: person.uuid.clone(),
message: "Person properties update would exceed the size limit and was \
rejected"
.to_string(),
});
return Err(Status::invalid_argument(format!(
"person properties update would exceed the size limit: {jsonb_size} bytes \
(jsonb) over the {} ceiling",
self.size_limits.threshold,
)));
}
}
let properties_bytes = serde_json::to_vec(&new_properties)
.map_err(|e| Status::internal(format!("serialize updated properties: {e}")))?;
// A version this pod already put on the wire is spent even when
// it never learned the outcome, so the next one has to clear that
// floor as well as the state it derived from. Reusing it produces
// a second record at the same version, and the writer's strict
// guard keeps only whichever arrived first.
let base_version = self
.emitted_versions
.floor_for(partition, &cache_key, person.version);
let updated_person = CachedPerson {
id: person.id,
uuid: person.uuid.clone(),
team_id: person.team_id,
properties: properties_bytes,
created_at: person.created_at,
version: base_version + 1,
is_identified: identified_now,
is_deleted: false,
last_seen_at: merged_last_seen,
};
let committed = self
.commit_document(partition, &cache_key, updated_person, guard, inflight_guard)
.await?;
counter!("personhog_leader_updates_total", "outcome" => "updated").increment(1);
Ok(Response::new(UpdatePersonPropertiesResponse {
person: Some(committed.proto),
updated: true,
}))
}
async fn raise_person_version(
&self,
request: Request<RaisePersonVersionRequest>,
) -> Result<Response<RaisePersonVersionResponse>, Status> {
let partition = partition_from_metadata(&request)?;
let req = request.into_inner();
self.validate_partition(partition, req.team_id, req.person_id)?;
if req.min_version <= 0 {
return Err(Status::invalid_argument("min_version must be positive"));
}
self.check_authority(partition)?;
let Some(inflight_guard) = self.inflight.try_begin(partition) else {
return Err(Status::failed_precondition(format!(
"partition {partition} is fenced for handoff; writes are rejected"
)));
};
let cache_key = PersonCacheKey {
team_id: req.team_id,
person_id: req.person_id,
};
let mutex = self
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let lock_wait = std::time::Instant::now();
let guard = mutex.lock_owned().await;
histogram!("personhog_leader_person_lock_wait_ms")
.record(lock_wait.elapsed().as_secs_f64() * 1000.0);
if !self.dirty_index.can_admit(&cache_key) {
counter!("personhog_leader_writes_shed_total", "reason" => "dirty_index_full")
.increment(1);
return Err(Status::resource_exhausted(
"dirty index at capacity: the writer is behind and this raise cannot be \
tracked; retry later",
));
}
if let Some(fence) = self.check_fence(&cache_key) {
counter!("personhog_leader_writes_fenced_total").increment(1);
if let Some(healer) = &self.fence_healer {
healer.maybe_heal(cache_key.clone(), fence);
}
return Err(fenced_status(&fence));
}
let person = self.lookup_or_load_locked(partition, &cache_key).await?;
self.check_authority(partition)?;
if person.is_deleted {
return Err(Status::not_found("person is destroyed"));
}
if person.version >= req.min_version {
counter!("personhog_leader_version_raises_total", "outcome" => "at_or_above")
.increment(1);
return self.authoritative_ok(
partition,
RaisePersonVersionResponse {
person: Some(cached_person_into_proto(person)),
raised: false,
},
);
}
// A spent version may already be in the changelog, so the raise must clear it.
let floor = self
.emitted_versions
.floor_for(partition, &cache_key, person.version);
let version = floor
.checked_add(1)
.ok_or_else(|| Status::invalid_argument("no version left above the emitted floor"))?
.max(req.min_version);
let mut raised = CachedPerson { version, ..person };
// A stored row over the size limit would stop the writer's partition, so trim it as an update does.
let stored = raised
.parse_properties()
.map_err(|e| Status::internal(format!("cached properties unparseable: {e}")))?;
let stored_size = jsonb_column_size(&stored);
if stored_size >= self.size_limits.threshold {
match trim_properties_to_fit_size(&stored, self.size_limits.trim_target) {
TrimResult::Trimmed(trimmed) => {
counter!("personhog_leader_properties_trimmed_total").increment(1);
self.warnings.emit(&SizeViolationWarning {
team_id: cache_key.team_id,
person_uuid: raised.uuid.clone(),
message: "Oversized person properties were trimmed to fit the size limit"
.to_string(),
});
raised.properties = serde_json::to_vec(&trimmed).map_err(|e| {
Status::internal(format!("serialize trimmed properties: {e}"))
})?;
}
TrimResult::Fits => {}
TrimResult::CannotFit => {
counter!(
"personhog_leader_version_raises_total",
"outcome" => "rejected_unremediable"
)
.increment(1);
return Err(semantic_refusal(
format!(
"person properties exceed the size limit: stored state is \
{stored_size} bytes (jsonb) and protected properties alone exceed \
the {} trim target",
self.size_limits.trim_target,
),
"person-too-large",
));
}
}
}
let committed = self
.commit_document(partition, &cache_key, raised, guard, inflight_guard)
.await?;
counter!("personhog_leader_version_raises_total", "outcome" => "raised").increment(1);
self.authoritative_ok(
partition,
RaisePersonVersionResponse {
person: Some(committed.proto),
raised: true,
},
)
}
async fn fold_person_document(
&self,
request: Request<FoldPersonDocumentRequest>,
) -> Result<Response<FoldPersonDocumentResponse>, Status> {
let partition = partition_from_metadata(&request)?;
let req = request.into_inner();
self.validate_partition(partition, req.team_id, req.person_id)?;
self.check_authority(partition)?;
let op_id = Uuid::parse_str(&req.op_id)
.map_err(|_| Status::invalid_argument("op_id must be a valid UUID"))?;
if req.sealed_snapshots.is_empty() {
return Err(Status::invalid_argument(
"sealed_snapshots must be non-empty: a merge with no sealed sources has nothing \
to fold",
));
}
let Some(inflight_guard) = self.inflight.try_begin(partition) else {
return Err(Status::failed_precondition(format!(
"partition {partition} is fenced for handoff; writes are rejected"
)));
};
// Parse and sanitize every JSON input before taking the per-key
// lock. Snapshot properties were sanitized when the source's
// leader cached them, but they crossed the wire since; sanitizing
// again keeps the fold's admission guarantee self-contained.
let mut sanitize_totals = SanitizeStats::default();
let mut track = |stats: SanitizeStats| {
sanitize_totals.nul_strings += stats.nul_strings;
sanitize_totals.clamped_numbers += stats.clamped_numbers;
};
let mut event_set = parse_json_object_field(&req.event_set, "event_set")?;
let mut event_set_once = parse_json_object_field(&req.event_set_once, "event_set_once")?;
track(sanitize_for_jsonb(&mut event_set));
track(sanitize_for_jsonb(&mut event_set_once));
let (snapshots, snapshot_stats) =
parse_sealed_snapshots(&req.sealed_snapshots, req.team_id, req.person_id)?;
track(snapshot_stats);
let cache_key = PersonCacheKey {
team_id: req.team_id,
person_id: req.person_id,
};
let mutex = self
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let lock_wait = std::time::Instant::now();
let guard = mutex.lock_owned().await;
histogram!("personhog_leader_person_lock_wait_ms")
.record(lock_wait.elapsed().as_secs_f64() * 1000.0);
if !self.dirty_index.can_admit(&cache_key) {
counter!("personhog_leader_writes_shed_total", "reason" => "dirty_index_full")
.increment(1);
return Err(Status::resource_exhausted(
"dirty index at capacity: the writer is behind and this fold cannot be tracked; \
retry later",
));
}
// The merge target is marked, never fenced, so a fence here is
// either a ghost from a settled op (the healer clears it and the
// saga's retry goes through) or a bug. Same-op tolerance mirrors
// FencePerson's re-seal semantics.
if let Some(fence) = self.check_fence(&cache_key) {
if fence.op_id != op_id {
counter!("personhog_leader_writes_fenced_total").increment(1);
if let Some(healer) = &self.fence_healer {
healer.maybe_heal(cache_key.clone(), fence);
}
return Err(fenced_status(&fence));
}
}
let person = self.lookup_or_load_locked(partition, &cache_key).await?;
// The load can park past the claim, and a not-found here is a
// verdict the router delivers, so it needs ownership like any
// answer.
self.check_authority(partition)?;
if person.is_deleted {
return Err(Status::not_found("person is destroyed"));
}
// The mark row — the fence's source of truth — must vouch for the
// op holding this person as its live merge target before the fold
// may write. The fence check above proves nothing here (the target
// is marked, never fenced), and without this a superseded or
// settled saga runner's late fold would still land. Mirrors
// ReleaseFence: unverifiable requests are refused — fail closed.
let Some(fallback) = &self.fallback else {
return Err(semantic_refusal(
"no lifecycle database configured; refusing to fold",
"no-lifecycle-db",
));
};
match target_mark_status(fallback, op_id, req.team_id, req.person_id).await {
Ok(Some(status)) if status == "marked" => {}
Ok(_) => {
counter!("personhog_leader_fences_total", "action" => "fold_unverified")
.increment(1);
return Err(semantic_refusal(
"op holds no live target mark for this person; refusing to fold",
"fold-unverified",
));
}
Err(e) => {
tracing::warn!(error = %e, "target mark verification failed; rejecting (fail closed)");
return Err(Status::unavailable(
"could not verify the lifecycle op against its target mark; retry",
));
}
}
// The fold: the target wins every key it has; snapshots fill
// still-absent keys in request order; then the merge event's $set
// overrides and $set_once fills. All inputs are sanitized, so the
// merged document is measured in stored form.
let mut target_properties = match person.parse_properties() {
Ok(value) if value.is_object() => value,
Ok(_) => serde_json::Value::Object(serde_json::Map::new()),
Err(e) => {
return Err(Status::internal(format!(
"cached properties unparseable: {e}"
)))
}
};
// The cached state is an input like any other: rows loaded from
// Postgres or warmed from records that predate sanitization can
// carry dirt the wire inputs cannot.
track(sanitize_for_jsonb(&mut target_properties));
if sanitize_totals.nul_strings > 0 {
counter!("personhog_leader_properties_nul_sanitized_total")
.increment(sanitize_totals.nul_strings);
}
if sanitize_totals.clamped_numbers > 0 {
counter!("personhog_leader_properties_numbers_clamped_total")
.increment(sanitize_totals.clamped_numbers);
}
let target_map = target_properties
.as_object()
.expect("target_properties was just normalized to an object");
let mut folded = target_properties.clone();
let folded_map = folded
.as_object_mut()
.expect("folded clones the normalized target");
for snapshot in &snapshots {
if let Some(map) = snapshot.properties.as_object() {
for (key, value) in map {
if !folded_map.contains_key(key) {
folded_map.insert(key.clone(), value.clone());
}
}
}
}
if let Some(map) = event_set.as_object() {
for (key, value) in map {
folded_map.insert(key.clone(), value.clone());
}
}
if let Some(map) = event_set_once.as_object() {
for (key, value) in map {
if !folded_map.contains_key(key) {
folded_map.insert(key.clone(), value.clone());
}
}
}
// Unlike a property update, a fold cannot be rejected for size:
// the saga would re-drive it forever, so trimming is the only
// completing behavior. Trim candidates are the fold's own
// contribution — keys the target did not already hold — so a
// within-limit target never loses a key it had to a fold. The
// target's own keys join the candidates only when its stored
// document already exceeded the limit (the remediation the update
// path applies to already-oversized rows). The trim aims for the
// hysteresis target first and, when that is unreachable, retries
// against the hard ceiling: any document at or under the
// threshold is still applyable by the writer. When neither bound
// is reachable, the fold keeps the target's properties,
// discarding its contribution.
let jsonb_size = jsonb_column_size(&folded);
let mut fold_outcome = "folded";
if jsonb_size > self.size_limits.threshold {
let target_size = jsonb_column_size(&target_properties);
let target_oversized = target_size >= self.size_limits.threshold;
let mut candidates: Vec<String> = folded
.as_object()
.expect("folded is an object")
.keys()
.filter(|k| !target_map.contains_key(*k) && can_trim_property(k))
.cloned()
.collect();
candidates.sort();
if target_oversized {
let mut own: Vec<String> = target_map
.keys()
.filter(|k| can_trim_property(k))
.cloned()
.collect();
own.sort();
candidates.extend(own);
}
let trim_result = match trim_properties_with_candidates(
&folded,
self.size_limits.trim_target,
candidates.clone(),
) {
TrimResult::CannotFit => {
trim_properties_with_candidates(&folded, self.size_limits.threshold, candidates)
}
result => result,
};
match trim_result {
TrimResult::Trimmed(trimmed) => {
counter!("personhog_leader_properties_trimmed_total").increment(1);
self.warnings.emit(&SizeViolationWarning {
team_id: cache_key.team_id,
person_uuid: person.uuid.clone(),
message: "Merged person properties exceeded the size limit and were \
trimmed to fit"
.to_string(),
});
folded = trimmed;
}
TrimResult::Fits => {}
TrimResult::CannotFit => {
self.warnings.emit(&SizeViolationWarning {
team_id: cache_key.team_id,
person_uuid: person.uuid.clone(),
message: "Merged person properties exceed the size limit and could not \
be trimmed; the merged-in properties were discarded"
.to_string(),
});
if target_oversized {
// No applyable document exists: the stored one
// itself violates the size constraint (protected
// keys alone exceed the ceiling). Producing it
// anyway would halt the writer — admission
// promises every acked record applies verbatim,
// and the writer fail-stops on a violation rather
// than skip — and rejecting would wedge the
// saga's re-drive loop. The fold completes
// without producing: this person's fold effects
// (properties and scalars) are skipped. Accepted
// residual — README, "Admission".
counter!("personhog_leader_folds_total", "outcome" => "unapplyable")
.increment(1);
tracing::error!(
team_id = cache_key.team_id,
person_uuid = %person.uuid,
stored_size = target_size,
threshold = self.size_limits.threshold,
"fold target's stored properties exceed the size constraint; \
skipping the fold's document write"
);
return self.authoritative_ok(
partition,
FoldPersonDocumentResponse {
person: Some(cached_person_into_proto(person)),
},
);
}
fold_outcome = "unremediable";
folded = target_properties.clone();
}
}
}
// Scalars: created_at is the min over the target and every
// snapshot (ignoring non-positive values a malformed snapshot
// could carry); is_identified is unconditionally true — a merge
// is an identify; last_seen_at max-merges like the update path —
// the merged person was last seen whenever any constituent was
// (snapshot values were already hour-floored when stored).
//
// The Postgres backend never passes last_seen_at to its merge
// update; the caller's follow-up update advances it.
let created_at = snapshots
.iter()
.map(|snapshot| snapshot.created_at)
.filter(|ts| *ts > 0)
.chain(std::iter::once(person.created_at))
.min()
.unwrap_or(person.created_at);
let last_seen_at = snapshots
.iter()
.filter_map(|snapshot| snapshot.last_seen_at)
.chain(person.last_seen_at)
.max();
// The version is a max-merge over the target's floor and every
// sealed version, plus one. A source's death document starts
// from the same sealed + 1, so the fold sits at or above it and
// equals it when the highest sealed version dominates. A source
// row that moved after its seal raises its death document above
// the fold. The streams are per-person, so the relation is a
// convention, not a collision guard. Re-applying the fold only
// bumps the version again, so it converges under at-least-once
// delivery.
let base_version = self
.emitted_versions
.floor_for(partition, &cache_key, person.version);
let max_sealed = snapshots
.iter()
.map(|snapshot| snapshot.version)
.max()
.unwrap_or(0);
let version = base_version.max(max_sealed).checked_add(1).ok_or_else(|| {
Status::invalid_argument("sealed versions leave no room for the folded version")
})?;
let folded_bytes = serde_json::to_vec(&folded)
.map_err(|e| Status::internal(format!("serialize folded properties: {e}")))?;
let folded_person = CachedPerson {
id: person.id,
uuid: person.uuid.clone(),
team_id: person.team_id,
properties: folded_bytes,
created_at,
version,
is_identified: true,
is_deleted: false,
last_seen_at,
};
let committed = self
.commit_document(partition, &cache_key, folded_person, guard, inflight_guard)
.await?;
counter!("personhog_leader_folds_total", "outcome" => fold_outcome).increment(1);
self.authoritative_ok(
partition,
FoldPersonDocumentResponse {
person: Some(committed.proto),
},
)
}
async fn fence_person(
&self,
request: Request<FencePersonRequest>,
) -> Result<Response<FencePersonResponse>, Status> {
self.fence_person_rpc(request).await
}
async fn fence_persons(
&self,
request: Request<FencePersonsRequest>,
) -> Result<Response<FencePersonsResponse>, Status> {
self.fence_persons_rpc(request).await
}
async fn release_fence(
&self,
request: Request<ReleaseFenceRequest>,
) -> Result<Response<ReleaseFenceResponse>, Status> {
self.release_fence_rpc(request).await
}
async fn release_fences(
&self,
request: Request<ReleaseFencesRequest>,
) -> Result<Response<ReleaseFencesResponse>, Status> {
self.release_fences_rpc(request).await
}
}
/// Remove lock entries that no one is currently waiting on. Entries
/// with `Arc::strong_count == 1` are only held by the map itself, so
/// no request is actively using them. Returns the number removed.
pub fn sweep_idle_locks(locks: &DashMap<PersonCacheKey, Arc<Mutex<()>>>) -> usize {
let before = locks.len();
locks.retain(|_, v| Arc::strong_count(v) > 1);
let removed = before - locks.len();
if removed > 0 {
tracing::debug!(removed, remaining = locks.len(), "swept idle locks");
}
removed
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::time::Duration;
use chrono::Datelike;
use common_kafka::config::KafkaConfig;
use envconfig::Envconfig;
use personhog_proto::personhog::types::v1::{LifecycleOpType, ReleaseOutcome};
use tonic::Code;
use super::*;
use crate::recovery::RecoveryConfig;
/// A client's zero `DateTime` serialises as year 1, and persons created
/// from such events before personhog carry it as `created_at`.
const YEAR_ONE_MS: i64 = -62_135_596_800_000;
#[test]
fn writeable_accepts_what_the_writer_binds_and_refuses_the_rest() {
let person = CachedPerson {
id: 1,
uuid: "00000000-0000-0000-0000-000000000001".to_string(),
team_id: 7,
properties: b"{}".to_vec(),
created_at: YEAR_ONE_MS,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: Some(YEAR_ONE_MS),
};
assert_eq!(assert_writeable(&person), Ok(()));
assert!(
Utc.timestamp_millis_opt(YEAR_ONE_MS).single().is_some(),
"the writer converts with the same call, so year 1 binds"
);
// The bounds are Postgres's, not chrono's: both ends convert, and
// one millisecond past either end is refused.
let floor = Utc.timestamp_millis_opt(PG_TIMESTAMPTZ_MIN_MS).unwrap();
assert_eq!(
(floor.year(), floor.month(), floor.day()),
(-4713, 11, 24),
"Julian day 0 in the proleptic Gregorian calendar"
);
assert!(Utc
.timestamp_millis_opt(MAX_EPOCH_MS_YEAR_9999)
.single()
.is_some());
assert!(writeable_epoch_ms(PG_TIMESTAMPTZ_MIN_MS));
assert!(writeable_epoch_ms(MAX_EPOCH_MS_YEAR_9999));
assert!(!writeable_epoch_ms(PG_TIMESTAMPTZ_MIN_MS - 1));
assert!(!writeable_epoch_ms(MAX_EPOCH_MS_YEAR_9999 + 1));
for (field, created_at, last_seen_at) in [
("created_at", i64::MIN, None),
("created_at", PG_TIMESTAMPTZ_MIN_MS - 1, None),
("created_at", i64::MAX, None),
("last_seen_at", YEAR_ONE_MS, Some(-300_000_000_000_000)),
("last_seen_at", YEAR_ONE_MS, Some(i64::MAX)),
] {
let refused = CachedPerson {
created_at,
last_seen_at,
..person.clone()
};
let reason = assert_writeable(&refused).expect_err(field);
assert!(reason.starts_with(field), "{reason}");
}
}
#[test]
fn last_seen_floors_to_the_hour_on_both_sides_of_the_epoch() {
assert_eq!(floor_to_hour(3_600_000 + 1), 3_600_000);
assert_eq!(floor_to_hour(YEAR_ONE_MS + 1234), YEAR_ONE_MS);
assert_eq!(floor_to_hour(-1), -MS_PER_HOUR);
}
fn make_key(team_id: i64, person_id: i64) -> PersonCacheKey {
PersonCacheKey { team_id, person_id }
}
fn wire_snapshot(person: Person, ordinal: i32) -> SealedSourceSnapshot {
SealedSourceSnapshot {
person: Some(person),
ordinal,
}
}
fn source_person(id: i64, properties: &serde_json::Value) -> Person {
Person {
id,
team_id: 7,
properties: serde_json::to_vec(properties).unwrap(),
..Default::default()
}
}
#[test]
fn sealed_snapshot_admission_rejects_implausible_sources() {
let valid = serde_json::json!({});
let cases: Vec<(&str, Vec<SealedSourceSnapshot>)> = vec![
(
"missing person",
vec![SealedSourceSnapshot {
person: None,
ordinal: 0,
}],
),
(
"wrong team",
vec![wire_snapshot(
Person {
team_id: 8,
..source_person(2, &valid)
},
0,
)],
),
(
"snapshot is the target",
vec![wire_snapshot(source_person(1, &valid), 0)],
),
(
"death document",
vec![wire_snapshot(
Person {
is_deleted: true,
..source_person(2, &valid)
},
0,
)],
),
(
"duplicate ordinal",
vec![
wire_snapshot(source_person(2, &valid), 0),
wire_snapshot(source_person(3, &valid), 0),
],
),
(
"non-object properties",
vec![wire_snapshot(
Person {
properties: b"[1]".to_vec(),
..source_person(2, &valid)
},
0,
)],
),
];
for (label, sealed) in cases {
let status = parse_sealed_snapshots(&sealed, 7, 1).expect_err(label);
assert_eq!(status.code(), Code::InvalidArgument, "{label}");
}
}
#[test]
fn sealed_snapshot_admission_sorts_by_ordinal_and_counts_sanitization() {
let nul_dirty = serde_json::json!({"a": "x\u{0000}y"});
let float_dirty = serde_json::json!({"b": 1e308});
let (snapshots, stats) = parse_sealed_snapshots(
&[
wire_snapshot(
Person {
version: 9,
..source_person(2, &nul_dirty)
},
1,
),
wire_snapshot(
Person {
version: 4,
..source_person(3, &float_dirty)
},
0,
),
],
7,
1,
)
.expect("plausible snapshots admit");
assert_eq!(
snapshots
.iter()
.map(|snapshot| snapshot.version)
.collect::<Vec<_>>(),
vec![4, 9],
"returned in ordinal order, not request order"
);
assert_eq!(stats.nul_strings, 1);
assert_eq!(stats.clamped_numbers, 1);
}
/// A service with no PG pool and no reachable Kafka — enough to
/// exercise the miss path, where no test reaches Kafka or PG.
async fn make_test_service() -> PersonHogLeaderService {
let kafka = KafkaConfig::init_from_hashmap(&HashMap::new()).unwrap();
PersonHogLeaderService::new(
Arc::new(PartitionedCache::new(16)),
None,
Arc::new(DashMap::new()),
Arc::new(InflightTracker::new()),
// Partition 0 served under one partition.
Arc::new(DashMap::from_iter([(
0,
crate::coordination::Capture {
cluster: None,
topic: Arc::from("personhog_updates"),
config: personhog_coordination::types::ChangelogConfig::new(
"personhog_updates",
1,
),
},
)])),
Arc::new(DirtyIndex::new(16)),
Arc::new(
ChangelogRecovery::new(RecoveryConfig {
kafka,
pod_name: "test".to_string(),
recv_timeout: Duration::from_millis(10),
pool_size: 1,
})
.expect("build recovery pool"),
),
PropertySizeLimits::new(655360, 524288),
WarningsProducer::disabled(),
Arc::new(DashMap::new()),
// Lazily built: nothing here connects until a write acquires
// a partition, which these tests never do.
Arc::new(FencedChangelogProducers::new(FencedProducerConfig {
kafka: KafkaConfig::init_from_hashmap(&HashMap::new()).unwrap(),
init_timeout: Duration::from_secs(1),
commit_timeout: Duration::from_secs(1),
broker_txn_timeout: Duration::from_secs(10),
window: Duration::from_millis(5),
window_max_writes: 32,
settle_budget: Duration::from_secs(1),
lanes: 1,
})),
test_authority(),
Arc::new(EmittedVersions::new(1_000_000)),
)
}
/// A claim that stays valid for the whole of any test.
fn test_authority() -> Arc<AuthorityClock> {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(3600), Instant::now());
clock
}
/// The guarantee the clock exists for: a pod whose renewals have
/// stopped refuses to answer as the partition's owner, without
/// anything running to notice they stopped. The keepalive being
/// wedged is exactly the case the lease machinery cannot cover.
#[tokio::test]
async fn a_pod_whose_renewals_stopped_refuses_to_serve() {
// A clock whose renewals stopped longer ago than the margin —
// constructed stale rather than aged by sleeping, so no runner
// pace can blur which side of the margin the test is on.
let margin = Duration::from_secs(20);
let clock = Arc::new(AuthorityClock::stale_for(
margin,
margin + Duration::from_secs(1),
));
let service = PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
};
let err = service
.check_authority(0)
.expect_err("a lapsed lease must not serve");
assert_eq!(err.code(), Code::FailedPrecondition);
clock.confirm(Instant::now());
service
.check_authority(0)
.expect("a confirmed renewal restores service");
}
/// Losing the lease is decided immediately, not after the margin:
/// the coordinator may already be reassigning.
#[tokio::test]
async fn surrendering_the_lease_stops_reads_at_once() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
};
service.check_authority(0).expect("still the owner");
clock.surrender();
assert_eq!(
service.check_authority(0).unwrap_err().code(),
Code::FailedPrecondition
);
}
/// The gate has to be wired into the RPC, not merely implemented:
/// this drives `get_person` itself rather than the check in
/// isolation.
///
/// It does not distinguish the two checks — a read that finds its
/// person in cache never waits, so either one refuses it, and both
/// return the same status. `a_read_admitted_before_the_lapse_still_
/// refuses_to_answer` is what pins the second.
#[tokio::test]
async fn get_person_refuses_once_authority_lapses() {
let margin = Duration::from_secs(20);
let clock = Arc::new(AuthorityClock::stale_for(
margin,
margin + Duration::from_secs(1),
));
let service = PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
};
// The fixture's single partition makes 0 the only routing answer.
let (team_id, person_id) = (7, 42);
service.cache.create_partition(0);
service.cache.put(
0,
PersonCacheKey { team_id, person_id },
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
let request = || {
let mut request = Request::new(GetPersonRequest {
team_id,
person_id,
read_options: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
let err = service
.get_person(request())
.await
.expect_err("a lapsed lease must refuse the read");
assert_eq!(err.code(), Code::FailedPrecondition);
clock.confirm(Instant::now());
service
.get_person(request())
.await
.expect("a confirmed renewal serves the read");
}
/// Wiring pin for `fence_person`: with both gates removed the fence
/// installs. Either gate alone satisfies it; the parked and
/// refused-before-work pins each single out one.
#[tokio::test]
async fn fence_person_refuses_once_authority_lapses() {
let margin = Duration::from_secs(20);
let clock = Arc::new(AuthorityClock::stale_for(
margin,
margin + Duration::from_secs(1),
));
let service = PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
};
let (team_id, person_id) = (7, 42);
service.cache.create_partition(0);
service.cache.put(
0,
PersonCacheKey { team_id, person_id },
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
let request = || {
let mut request = Request::new(FencePersonRequest {
team_id,
person_id,
op_id: "11111111-1111-1111-1111-111111111111".to_string(),
op_type: LifecycleOpType::Delete as i32,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
let err = service
.fence_person(request())
.await
.expect_err("a lapsed lease must not install a fence");
assert_eq!(err.code(), Code::FailedPrecondition);
clock.confirm(Instant::now());
service
.fence_person(request())
.await
.expect("a confirmed renewal installs the fence");
}
/// `release_fence`'s aborted arm acks without producing, so
/// `commit_document`'s re-check never runs; either release gate may
/// refuse here — the timeout test is the admission-specific pin.
#[tokio::test]
async fn release_fence_refuses_once_authority_lapses() {
let margin = Duration::from_secs(20);
let clock = Arc::new(AuthorityClock::stale_for(
margin,
margin + Duration::from_secs(1),
));
let service = PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
};
let (team_id, person_id) = (7, 42);
service.cache.create_partition(0);
let request = || {
let mut request = Request::new(ReleaseFenceRequest {
team_id,
person_id,
op_id: "11111111-1111-1111-1111-111111111111".to_string(),
outcome: ReleaseOutcome::Aborted as i32,
..Default::default()
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
let err = service
.release_fence(request())
.await
.expect_err("a lapsed lease must not ack a release");
assert_eq!(err.code(), Code::FailedPrecondition);
clock.confirm(Instant::now());
service
.release_fence(request())
.await
.expect("a confirmed renewal acks the abort");
}
/// A fence admitted before the lapse must not install once parked
/// past it. The map assertion is the point: the install, not the
/// status, is the danger.
#[tokio::test]
async fn a_fence_admitted_before_the_lapse_does_not_install() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
service.cache.create_partition(0);
// Seeded live, so a confirmed clock would seal successfully; the
// refusal must come from the lapse alone.
service.cache.put(
0,
cache_key.clone(),
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
// Hold the per-key lock so the fence parks before its seal.
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
let mut request = Request::new(FencePersonRequest {
team_id,
person_id,
op_id: "11111111-1111-1111-1111-111111111111".to_string(),
op_type: LifecycleOpType::Delete as i32,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let fencing = tokio::spawn({
let service = Arc::clone(&service);
async move { service.fence_person(request).await }
});
tokio::task::yield_now().await;
clock.surrender();
drop(held);
let err = fencing
.await
.unwrap()
.expect_err("a fence parked past the lapse must refuse");
assert_eq!(err.code(), Code::FailedPrecondition);
assert!(
service.fences.is_empty(),
"no fence may install under a lapsed claim"
);
}
/// A release admitted while owned must not ack once parked past an
/// ordinary handoff: the partition (and its fences) are gone, so the
/// ack would report a removal that never happened. The lease stays
/// valid throughout — this pins the ownership re-check, not the clock.
#[tokio::test]
async fn a_release_parked_past_a_handoff_does_not_ack() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
let op_id = Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap();
service.cache.create_partition(0);
service.fences.insert(
cache_key.clone(),
FenceState {
op_id,
op_type: LifecycleOpType::Delete,
sealed_at: None,
},
);
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
let mut request = Request::new(ReleaseFenceRequest {
team_id,
person_id,
op_id: op_id.to_string(),
outcome: ReleaseOutcome::Aborted as i32,
..Default::default()
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let releasing = tokio::spawn({
let service = Arc::clone(&service);
async move { service.release_fence(request).await }
});
tokio::task::yield_now().await;
// The handoff completes while the release is parked.
service.cache.drop_partition(0);
crate::fence::drop_partition_fences(&service.fences, 0, 1, |_| false);
drop(held);
let err = releasing
.await
.unwrap()
.expect_err("a release parked past a handoff must refuse");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// A release admitted before the lapse must not ack once parked past
/// it: removing only this stale map's entry while the saga believes
/// the fence released would be a vacuous success.
#[tokio::test]
async fn a_release_admitted_before_the_lapse_does_not_ack() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
let op_id = Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap();
service.cache.create_partition(0);
service.fences.insert(
cache_key.clone(),
FenceState {
op_id,
op_type: LifecycleOpType::Delete,
sealed_at: None,
},
);
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
let mut request = Request::new(ReleaseFenceRequest {
team_id,
person_id,
op_id: op_id.to_string(),
outcome: ReleaseOutcome::Aborted as i32,
..Default::default()
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let releasing = tokio::spawn({
let service = Arc::clone(&service);
async move { service.release_fence(request).await }
});
tokio::task::yield_now().await;
clock.surrender();
drop(held);
let err = releasing
.await
.unwrap()
.expect_err("a release parked past the lapse must refuse");
assert_eq!(err.code(), Code::FailedPrecondition);
assert!(
service.fences.contains_key(&cache_key),
"the fence must survive a refused release"
);
}
/// The admission gates on both fence RPCs refuse before any work;
/// only the wait distinguishes them from the later re-checks, so a
/// held per-key lock turns a dropped admission gate into a timeout.
#[tokio::test]
async fn fence_rpcs_arriving_after_the_lapse_are_refused_at_admission() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
service.cache.create_partition(0);
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let _held = mutex.lock().await;
clock.surrender();
let mut fence = Request::new(FencePersonRequest {
team_id,
person_id,
op_id: "11111111-1111-1111-1111-111111111111".to_string(),
op_type: LifecycleOpType::Delete as i32,
});
fence
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let err = tokio::time::timeout(Duration::from_millis(200), service.fence_person(fence))
.await
.expect("the fence must be refused at admission, not behind the lock")
.expect_err("a lapsed claim must not fence");
assert_eq!(err.code(), Code::FailedPrecondition);
let mut release = Request::new(ReleaseFenceRequest {
team_id,
person_id,
op_id: "11111111-1111-1111-1111-111111111111".to_string(),
outcome: ReleaseOutcome::Aborted as i32,
..Default::default()
});
release
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let err = tokio::time::timeout(Duration::from_millis(200), service.release_fence(release))
.await
.expect("the release must be refused at admission, not behind the lock")
.expect_err("a lapsed claim must not release");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// A no-change update answers cached state, which is a read: parked
/// past the lapse it must refuse rather than answer updated:false.
#[tokio::test]
async fn a_no_change_update_refuses_once_parked_past_the_lapse() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
service.cache.create_partition(0);
service.cache.put(
0,
cache_key.clone(),
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({"plan": "pro"})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
// The same value the cache holds, so the no-change answer path runs.
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id,
person_id,
event_name: "$pageview".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"plan": "pro"})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let updating = tokio::spawn({
let service = Arc::clone(&service);
async move { service.update_person_properties(request).await }
});
tokio::task::yield_now().await;
clock.surrender();
drop(held);
let err = updating
.await
.unwrap()
.expect_err("a no-change answer parked past the lapse must refuse");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// A fold parked past the lapse must refuse, not answer: its
/// not-found is a verdict the router delivers to the saga, unlike
/// the FailedPrecondition a refusal bounces.
#[tokio::test]
async fn a_fold_parked_past_the_lapse_refuses_instead_of_answering_destroyed() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
service.cache.create_partition(0);
// A death document, so without the gate the fold answers a
// definitive not-found from a claim it no longer holds.
service.cache.put(
0,
cache_key.clone(),
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: b"{}".to_vec(),
created_at: 0,
version: 3,
is_identified: false,
is_deleted: true,
last_seen_at: None,
},
);
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
let mut request = Request::new(FoldPersonDocumentRequest {
team_id,
person_id,
op_id: "11111111-1111-1111-1111-111111111111".to_string(),
sealed_snapshots: vec![wire_snapshot(source_person(43, &serde_json::json!({})), 0)],
event_set: b"{}".to_vec(),
event_set_once: b"{}".to_vec(),
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let folding = tokio::spawn({
let service = Arc::clone(&service);
async move { service.fold_person_document(request).await }
});
tokio::task::yield_now().await;
clock.surrender();
drop(held);
let err = folding
.await
.unwrap()
.expect_err("a fold parked past the lapse must refuse");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// A write is serving too, and the lease-loss path surrenders before
/// it drains — so between the surrender and the local fence landing,
/// this check is what stops a pod that no longer holds its lease from
/// acking a mutation the successor will never see. Fencing covers the
/// same ground when it is on, but it cannot be enabled first, so this
/// is the only cover the intermediate rollout state has.
///
/// Like its read-path counterpart it does not distinguish admission
/// from the pre-produce re-check; `a_write_admitted_before_the_lapse_
/// is_not_produced` pins the second.
///
/// The person is seeded deliberately: without it a removed check
/// would still surface `FailedPrecondition` from the ownership guard
/// further down, and the test would pass having proved nothing.
/// Filtered-only lanes answer updated=false with no write or produce;
/// force writes. A demotion regression would produce, hang on the
/// absent broker, and time out rather than pass.
#[tokio::test]
async fn filtered_only_update_answers_without_writing() {
let service = make_test_service().await;
let (team_id, person_id) = (7, 43);
service.cache.create_partition(0);
service.cache.put(
0,
PersonCacheKey { team_id, person_id },
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000008".to_string(),
team_id,
properties: serde_json::to_vec(
&serde_json::json!({"$current_url": "https://example.com/a"}),
)
.unwrap(),
created_at: 0,
version: 3,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
let request = |force: bool| {
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: force,
team_id,
person_id,
event_name: "$pageview".to_string(),
set_properties: serde_json::to_vec(
&serde_json::json!({"$current_url": "https://example.com/b"}),
)
.unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
let response = tokio::time::timeout(
Duration::from_secs(5),
service.update_person_properties(request(false)),
)
.await
.expect("a demoted update must answer without producing")
.expect("demoted update answers ok")
.into_inner();
assert!(!response.updated);
assert_eq!(
response.person.expect("carries the person").version,
3,
"a discarded change must not bump the version"
);
}
#[tokio::test]
async fn update_refuses_once_authority_is_surrendered() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
};
let (team_id, person_id) = (7, 42);
service.cache.create_partition(0);
service.cache.put(
0,
PersonCacheKey { team_id, person_id },
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
let request = || {
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id,
person_id,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"a": 1})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
request
};
// Losing the lease is decided at once, not after the margin, so
// no sleep is needed to reach the state that matters.
clock.surrender();
// Bounded on purpose: with the check gone the handler runs on to
// produce against a broker that is not there, so an unbounded
// await would report this regression as a hung test rather than a
// failing one.
let result = tokio::time::timeout(
Duration::from_secs(5),
service.update_person_properties(request()),
)
.await
.expect("the refusal must come from the claim check, not from a produce timeout");
let err = result.expect_err("a surrendered pod must not ack a write");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// `get_person` checks the claim twice, and asserting only on the
/// status code cannot tell the two apart: delete either and the other
/// still answers FailedPrecondition. This one pins the second.
///
/// The load is what makes it distinct. A read that finds the person
/// in cache never waits, so the admission check is the only one it
/// can trip; a read that has to wait — on the per-key lock, or on a
/// changelog recovery — can be admitted under a claim that is gone by
/// the time there is an answer, and it is the answer, not the
/// arrival, that has to be backed by ownership.
#[tokio::test]
async fn a_read_admitted_before_the_lapse_still_refuses_to_answer() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
service.cache.create_partition(0);
// Hold the per-key lock. The read misses the cache, reaches for
// this lock, and parks there — which is where a changelog
// recovery would leave it.
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
let mut request = Request::new(GetPersonRequest {
team_id,
person_id,
read_options: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let reading = tokio::spawn({
let service = Arc::clone(&service);
async move { service.get_person(request).await }
});
tokio::task::yield_now().await;
// The claim goes while the read is parked, and the answer it was
// waiting for arrives at the same time.
clock.surrender();
service.cache.put(
0,
cache_key.clone(),
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
drop(held);
let err = reading
.await
.expect("the read task must not panic")
.expect_err("a claim that lapsed during the load must not be answered");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// The write path checks the claim twice for the same reason the read
/// path does, and asserting only on the status code cannot tell them
/// apart. This one pins the second.
///
/// A write admitted under a valid claim can wait on the per-key lock
/// behind another produce, and on a changelog recovery — long enough
/// for a starved keepalive's stamp to age out, for the lease to
/// expire, and for a successor to warm past the point this record
/// would land. Acking it then is acked-write loss that needs only one
/// wedged pod, not a double zombie.
#[tokio::test]
async fn a_write_admitted_before_the_lapse_is_not_produced() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
service.cache.create_partition(0);
service.cache.put(
0,
cache_key.clone(),
CachedPerson {
id: person_id,
uuid: "00000000-0000-0000-0000-000000000007".to_string(),
team_id,
properties: serde_json::to_vec(&serde_json::json!({})).unwrap(),
created_at: 0,
version: 1,
is_identified: false,
is_deleted: false,
last_seen_at: None,
},
);
// Hold the per-key lock so the write is admitted and then parks,
// where a concurrent produce for the same person would leave it.
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let held = mutex.lock().await;
let mut request = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id,
person_id,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"a": 1})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
request
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let writing = tokio::spawn({
let service = Arc::clone(&service);
async move { service.update_person_properties(request).await }
});
tokio::task::yield_now().await;
// The claim goes while the write is parked, then the lock frees.
clock.surrender();
drop(held);
// Bounded: without the re-check the handler runs on to produce
// against a broker that is not there, and an unbounded await
// would report the regression as a hang rather than a failure.
let result = tokio::time::timeout(Duration::from_secs(5), writing)
.await
.expect("the refusal must come from the claim check, not a produce timeout")
.expect("the write task must not panic");
let err = result.expect_err("a claim that lapsed during the wait must not be produced");
assert_eq!(err.code(), Code::FailedPrecondition);
}
/// The admission checks refuse *before* the request does any work, and
/// only the wait distinguishes them from the pre-answer re-checks: a
/// request that reaches the per-key lock has already been admitted.
///
/// Holding that lock turns the difference into a deadline. With
/// admission intact neither call ever reaches it; without it they park
/// there and are refused only once the lock frees, having meanwhile
/// taken an inflight seat a handoff's drain must wait out and, on a
/// miss, driven a Postgres fallback or a changelog recovery for a
/// partition this pod no longer answers for.
#[tokio::test]
async fn a_request_arriving_after_the_lapse_is_refused_before_it_loads() {
let clock = Arc::new(AuthorityClock::unclaimed());
clock.begin_session(Duration::from_secs(30), Instant::now());
let service = Arc::new(PersonHogLeaderService {
authority: Arc::clone(&clock),
..make_test_service().await
});
let (team_id, person_id) = (7, 42);
let cache_key = PersonCacheKey { team_id, person_id };
// Deliberately unseeded: the read reaches the lock only on a
// miss, and a hit would let a dropped admission check still be
// caught by the pre-answer one, proving nothing.
service.cache.create_partition(0);
// Anything admitted parks here; nothing admitted ever arrives.
let mutex = service
.locks
.entry(cache_key.clone())
.or_default()
.value()
.clone();
let _held = mutex.lock().await;
clock.surrender();
let mut read = Request::new(GetPersonRequest {
team_id,
person_id,
read_options: None,
});
read.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let err = tokio::time::timeout(Duration::from_millis(200), service.get_person(read))
.await
.expect("the read must be refused at admission, not behind the load")
.expect_err("a lapsed claim must not be served");
assert_eq!(err.code(), Code::FailedPrecondition);
let mut write = Request::new(UpdatePersonPropertiesRequest {
force_update: false,
team_id,
person_id,
event_name: "$set".to_string(),
set_properties: serde_json::to_vec(&serde_json::json!({"a": 1})).unwrap(),
set_once_properties: vec![],
unset_properties: vec![],
is_identified: None,
last_seen_at: None,
});
write
.metadata_mut()
.insert("x-partition", "0".parse().unwrap());
let err = tokio::time::timeout(
Duration::from_millis(200),
service.update_person_properties(write),
)
.await
.expect("the write must be refused at admission, not behind the load")
.expect_err("a lapsed claim must not be admitted");
assert_eq!(err.code(), Code::FailedPrecondition);
}
#[test]
#[should_panic(expected = "must not exceed")]
fn inverted_size_limits_refuse_to_construct() {
PropertySizeLimits::new(655_360, 655_361);
}
#[tokio::test]
async fn unmarked_miss_fails_closed_once_partition_is_released() {
let service = make_test_service().await;
let key = make_key(1, 1);
// Owned and unmarked: PG is trusted (NOT_FOUND, since the test
// service has no pool).
service.cache.create_partition(0);
let err = service.recover_or_load(0, &key).await.unwrap_err();
assert_eq!(err.code(), Code::NotFound);
// Released mid-miss (cache dropped, then marks cleared — release
// order): the same lookup must fail closed rather than trust a
// possibly-stale PG row the cleared mark no longer guards.
service.cache.drop_partition(0);
service.dirty_index.clear_partition(0);
let err = service.recover_or_load(0, &key).await.unwrap_err();
assert_eq!(err.code(), Code::FailedPrecondition);
}
#[test]
fn sweep_removes_idle_entries() {
let locks = DashMap::new();
locks.insert(make_key(1, 1), Arc::new(Mutex::new(())));
locks.insert(make_key(1, 2), Arc::new(Mutex::new(())));
locks.insert(make_key(1, 3), Arc::new(Mutex::new(())));
let removed = sweep_idle_locks(&locks);
assert_eq!(removed, 3);
assert_eq!(locks.len(), 0);
}
#[test]
fn sweep_preserves_held_entries() {
let locks = DashMap::new();
locks.insert(make_key(1, 1), Arc::new(Mutex::new(())));
locks.insert(make_key(1, 2), Arc::new(Mutex::new(())));
locks.insert(make_key(1, 3), Arc::new(Mutex::new(())));
// Simulate an active holder cloning the Arc (as lookup_or_load does)
let _held = locks.get(&make_key(1, 2)).unwrap().clone();
let removed = sweep_idle_locks(&locks);
assert_eq!(removed, 2);
assert_eq!(locks.len(), 1);
assert!(locks.contains_key(&make_key(1, 2)));
}
#[test]
fn sweep_is_noop_when_empty() {
let locks: DashMap<PersonCacheKey, Arc<Mutex<()>>> = DashMap::new();
let removed = sweep_idle_locks(&locks);
assert_eq!(removed, 0);
}
}