Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

28.6 KB
Raw Blame
//! The lifecycle fence RPCs: a fence freezes a person for one op and
//! seals its state, and a release closes the fence with the op's outcome.
//! The single-call and batch handlers share the per-person helpers here.

use std::sync::Arc;
use std::time::Instant;

use futures::StreamExt;
use metrics::{counter, histogram};
use personhog_common::grpc::is_semantic_refusal;
use personhog_proto::personhog::types::v1::{
    FencePersonRequest, FencePersonResponse, FencePersonsRequest, FencePersonsResponse,
    FencedPersonSeal, LifecycleOpType, Person, ReleaseFenceRequest, ReleaseFenceResponse,
    ReleaseFencesRequest, ReleaseFencesResponse, ReleaseOutcome,
};
use tonic::{Request, Response, Status};
use uuid::Uuid;

use crate::cache::{CachedPerson, PersonCacheKey};
use crate::fence::{
    fenced_status, mark_status, semantic_refusal, FenceState, MarkSnapshot, MarkVerifier,
};
use crate::pg::PgFallback;

use super::{cached_person_into_proto, partition_from_metadata, PersonHogLeaderService};

/// The ceiling on persons per `FencePersons` or `ReleaseFences` call. The
/// lifecycle service splits its calls at this size, so a larger batch is a
/// caller bug, not load.
const MAX_LIFECYCLE_BATCH_SIZE: usize = 100;

/// Fences or releases of one batch call in flight at once. Above the
/// fallback pool's size, extra loads only queue at the pool while the
/// batch holds the partition's handoff drain.
const LIFECYCLE_BATCH_CONCURRENCY: usize = 16;

/// The one error a batch answers for: a semantic refusal is the final
/// answer for its person and must not hide behind a sibling's transient
/// error, which the saga would retry.
#[allow(clippy::result_large_err)]
fn first_batch_failure(failures: Vec<Status>) -> Result<(), Status> {
    let mut failures = failures.into_iter();
    let Some(first) = failures.next() else {
        return Ok(());
    };
    Err(failures
        .find(is_semantic_refusal)
        .filter(|_| !is_semantic_refusal(&first))
        .unwrap_or(first))
}

/// The per-person inputs of a committed release, validated the same way
/// whether they arrive alone or in a batch.
struct CommittedRelease {
    person_id: i64,
    person_uuid: String,
    sealed_version: i64,
    created_at: i64,
    /// The version the saga committed its Postgres tombstone at, when the
    /// saga recorded one; the death document is produced at or above it.
    tombstone_version: Option<i64>,
}

impl CommittedRelease {
    #[allow(clippy::result_large_err)]
    fn validate(
        person_id: i64,
        person_uuid: String,
        sealed_version: Option<i64>,
        created_at: i64,
        tombstone_version: Option<i64>,
    ) -> Result<Self, Status> {
        // 0 is a legitimate sealed version (a fresh stub's), which is why
        // the field is explicitly optional in the proto.
        let Some(sealed_version) = sealed_version else {
            return Err(Status::invalid_argument(
                "sealed_version is required for a committed release",
            ));
        };
        if sealed_version < 0 {
            return Err(Status::invalid_argument(
                "sealed_version must not be negative",
            ));
        }
        if created_at <= 0 {
            return Err(Status::invalid_argument(
                "created_at is required for a committed release",
            ));
        }
        if Uuid::parse_str(&person_uuid).is_err() {
            return Err(Status::invalid_argument(
                "person_uuid must be a valid UUID for a committed release",
            ));
        }
        // The saga tombstones at no less than sealed + 1; anything lower is
        // a caller bug, not a version the death document could honor.
        if tombstone_version.is_some_and(|version| version <= sealed_version) {
            return Err(Status::invalid_argument(
                "tombstone_version must be above sealed_version for a committed release",
            ));
        }
        Ok(Self {
            person_id,
            person_uuid,
            sealed_version,
            created_at,
            tombstone_version,
        })
    }
}

/// A mark lookup that failed is rejected retriably: the op may well hold
/// the person, but the leader could not prove it — fail closed.
fn mark_lookup_failed(e: sqlx::Error) -> Status {
    tracing::warn!(error = %e, "mark verification failed; rejecting (fail closed)");
    Status::unavailable("could not verify the lifecycle op against its mark; retry")
}

/// Per-phase wall time of a committed release; the RPC histogram alone
/// cannot separate the lock, the load, the mark check, and the produce.
fn record_release_phase(phase: &'static str, started: Instant) {
    histogram!("personhog_leader_release_phase_ms", "phase" => phase)
        .record(started.elapsed().as_secs_f64() * 1000.0);
}

fn no_lifecycle_db() -> Status {
    semantic_refusal(
        "no lifecycle database configured; refusing to produce a death document",
        "no-lifecycle-db",
    )
}

impl PersonHogLeaderService {
    #[allow(clippy::result_large_err)]
    fn lifecycle_db(&self) -> Result<&PgFallback, Status> {
        self.fallback.as_ref().ok_or_else(no_lifecycle_db)
    }

    #[allow(clippy::result_large_err)]
    fn mark_verifier(&self) -> Result<&MarkVerifier, Status> {
        self.mark_verifier.as_ref().ok_or_else(no_lifecycle_db)
    }

    /// The op's marks, read once per op per pod.
    async fn mark_snapshot(&self, op_id: Uuid) -> Result<Arc<MarkSnapshot>, Status> {
        let verifier = self.mark_verifier()?;
        let started = Instant::now();
        let snapshot = verifier.snapshot(op_id).await.map_err(mark_lookup_failed);
        record_release_phase("snapshot", started);
        snapshot
    }

    /// The committed half of a release for one person, verified against
    /// the op's snapshot or its own mark row. Shared by `ReleaseFence` and
    /// `ReleaseFences` so what destroys a person is decided in one place.
    async fn release_committed(
        &self,
        partition: u32,
        team_id: i64,
        op_id: Uuid,
        release: &CommittedRelease,
        snapshot: &MarkSnapshot,
        lifecycle_db: &PgFallback,
    ) -> Result<(), Status> {
        let cache_key = PersonCacheKey {
            team_id,
            person_id: release.person_id,
        };
        let mutex = self
            .locks
            .entry(cache_key.clone())
            .or_default()
            .value()
            .clone();
        let lock_started = Instant::now();
        let guard = mutex.lock_owned().await;
        record_release_phase("lock_wait", lock_started);

        // Releasing another op's fence would break that op's seal.
        let fence = self.fences.get(&cache_key).map(|entry| *entry.value());
        if let Some(fence) = fence.filter(|fence| fence.op_id != op_id) {
            return Err(fenced_status(&fence));
        }

        // Release must stay idempotent for the saga's retry and the
        // sweeper, so a person the leader cannot load anymore is
        // tolerated.
        let load_started = Instant::now();
        let loaded = self.lookup_or_load_locked(partition, &cache_key).await;
        record_release_phase("load", load_started);
        let current = match loaded {
            Ok(person) => Some(person),
            Err(status) if status.code() == tonic::Code::NotFound => None,
            Err(status) => return Err(status),
        };

        // Duplicate release: the death document already exists;
        // producing another would only bump the version. Gate
        // first — a refused release must leave the fence.
        if current.as_ref().is_some_and(|p| p.is_deleted) {
            self.assert_authoritative(partition)?;
            self.fences.remove(&cache_key);
            return Ok(());
        }

        // The request carries the identity a cold leader has nothing else
        // for; when the leader does hold the person, the two must agree, or
        // the death document would rewrite the row's uuid on its way out.
        if let Some(person) = &current {
            if person.uuid != release.person_uuid {
                return Err(semantic_refusal(
                    "person_uuid does not match the person being released",
                    "uuid-mismatch",
                ));
            }
        }

        // The op's mark must vouch before anything is destroyed; the fence
        // stands on the request's word alone, and unverifiable is refused.
        let mark = if fence.is_some_and(|fence| snapshot.vouches_for(&fence)) {
            snapshot
                .status(team_id, release.person_id)
                .map(str::to_owned)
        } else {
            let read_started = Instant::now();
            let mark = mark_status(lifecycle_db, op_id, team_id, release.person_id).await;
            record_release_phase("verify_mark", read_started);
            mark.map_err(mark_lookup_failed)?
        };
        match mark.as_deref() {
            // A live mark: the op holds the person; proceed.
            Some("marked") | Some("sealed") => {}
            // The mark already settled as deleted: this release
            // already happened and the tombstone is durable;
            // absorb the retry.
            Some("deleted") => {
                self.assert_authoritative(partition)?;
                self.fences.remove(&cache_key);
                return Ok(());
            }
            _ => {
                counter!("personhog_leader_fences_total", "action" => "release_unverified")
                    .increment(1);
                return Err(semantic_refusal(
                    "op holds no live mark for this person; \
                     refusing to produce a death document",
                    "release-unverified",
                ));
            }
        }

        if !self.dirty_index.can_admit(&cache_key) {
            counter!("personhog_leader_writes_shed_total", "reason" => "dirty_index_full")
                .increment(1);
            return Err(Status::resource_exhausted(
                "dirty index at capacity: the writer is behind and this death document \
                 cannot be tracked; retry later",
            ));
        }
        // The death version is sealed + 1, floored by the emitted floor and
        // the cached version in case a deposed or indeterminate produce spent
        // a higher version; a cold leader falls back to the request's seal.
        // The saga's committed tombstone version floors it too: a direct
        // Postgres writer can move the row past the seal while the fence
        // only stops leader writes, and a death document below the
        // tombstone would relay a live version over it to ClickHouse.
        let base_version = self.emitted_versions.floor_for(
            partition,
            &cache_key,
            current
                .as_ref()
                .map(|p| p.version)
                .unwrap_or(0)
                .max(release.sealed_version),
        );
        let death_version = base_version.checked_add(1).ok_or_else(|| {
            Status::invalid_argument("sealed_version leaves no room for the death version")
        })?;
        let floored_by_tombstone = release
            .tombstone_version
            .is_some_and(|tombstone_version| tombstone_version > death_version);
        let death_version = if floored_by_tombstone {
            release.tombstone_version.unwrap_or(death_version)
        } else {
            death_version
        };
        let death = CachedPerson {
            id: release.person_id,
            uuid: release.person_uuid.clone(),
            team_id,
            properties: b"{}".to_vec(),
            // The sealed value, not the cached one: cold and warm
            // leaders must produce the same document.
            created_at: release.created_at,
            version: death_version,
            is_identified: false,
            is_deleted: true,
            last_seen_at: None,
        };
        let produce_started = Instant::now();
        // The RPC's own guard admitted the batch; this one rides the
        // commit through its outcome.
        let inflight = self.inflight.begin(partition);
        let committed = self
            .commit_document(partition, &cache_key, death, guard, inflight)
            .await;
        record_release_phase("produce", produce_started);
        let _committed = committed?;
        // The death document stays cached while its mark stands, answering
        // not-found from memory; the prune-time settle drops it once the
        // writer confirms, and PG answers from then on.
        self.fences.remove(&cache_key);
        counter!("personhog_leader_fences_total", "action" => "released_committed").increment(1);
        // Its own counter, not a `fences_total` action: the release above is
        // already counted there, and a sum over `action` must count it once.
        if floored_by_tombstone {
            counter!("personhog_leader_death_documents_floored_by_tombstone_total").increment(1);
        }
        Ok(())
    }

    /// Fence one person and seal its state under its per-person lock,
    /// shared by `FencePerson` and `FencePersons`. The caller has admitted
    /// the partition: routing, authority, ownership, and the inflight guard.
    async fn fence_one(
        &self,
        partition: u32,
        cache_key: PersonCacheKey,
        op_id: Uuid,
        op_type: LifecycleOpType,
    ) -> Result<Person, Status> {
        let mutex = self
            .locks
            .entry(cache_key.clone())
            .or_default()
            .value()
            .clone();
        let _guard = mutex.lock().await;

        let refence = if let Some(entry) = self.fences.get(&cache_key) {
            if entry.op_id != op_id {
                let holder = *entry.value();
                drop(entry);
                // One op holds a person at a time; the loser backs off. The
                // holder may be a ghost whose op settled unheard, so kick the
                // lazy heal, since a quiet person gets no other caller.
                if let Some(healer) = &self.fence_healer {
                    healer.maybe_heal(cache_key.clone(), holder);
                }
                return Err(fenced_status(&holder));
            }
            true
        } else {
            false
        };

        // The map has no eviction, so a surge of ops is bounded by shedding
        // new fences; re-seals are exempt because refusing them frees
        // nothing. The saga's retry absorbs the backpressure.
        if !refence && self.fences.len() >= self.fence_map_max_entries {
            counter!("personhog_leader_fences_total", "action" => "shed_capacity").increment(1);
            return Err(Status::resource_exhausted(format!(
                "fence map at capacity ({} live fences); retry later",
                self.fence_map_max_entries
            )));
        }

        // The seal is the newest cached state under the write lock, raised
        // to the emitted floor: an indeterminate pre-fence write may have
        // spent a higher version. A same-op re-fence re-seals the same way.
        let person = self.lookup_or_load_locked(partition, &cache_key).await?;
        // The load can park on the per-key lock or a recovery; the seal,
        // not the arrival, must be backed by ownership.
        self.check_authority(partition)?;
        if person.is_deleted {
            return Err(Status::not_found("person is destroyed"));
        }

        let cached_version = person.version;
        let mut sealed = cached_person_into_proto(person);
        sealed.version = self
            .emitted_versions
            .floor_for(partition, &cache_key, cached_version);

        // A same-op re-fence leaves the fence alone: a fresh seal time would
        // turn a takeover fence into one the op's snapshot may vouch for.
        if !refence {
            self.fences.insert(
                cache_key,
                FenceState {
                    op_id,
                    op_type,
                    sealed_at: Some(Instant::now()),
                },
            );
        }
        counter!("personhog_leader_fences_total", "action" => "fenced").increment(1);
        Ok(sealed)
    }

    /// The aborted half of a release for one person: drop the fence, keep
    /// the entry, produce nothing. Gated first — a refused release must
    /// leave the fence (pinned).
    async fn release_aborted(
        &self,
        partition: u32,
        team_id: i64,
        person_id: i64,
        op_id: Uuid,
    ) -> Result<(), Status> {
        let cache_key = PersonCacheKey { team_id, person_id };
        let mutex = self
            .locks
            .entry(cache_key.clone())
            .or_default()
            .value()
            .clone();
        let _guard = mutex.lock().await;

        // Releasing another op's fence would break that op's seal.
        if let Some(entry) = self.fences.get(&cache_key) {
            if entry.op_id != op_id {
                return Err(fenced_status(entry.value()));
            }
        }
        self.assert_authoritative(partition)?;
        self.fences.remove(&cache_key);
        counter!("personhog_leader_fences_total", "action" => "released_aborted").increment(1);
        Ok(())
    }

    pub(super) async fn fence_person_rpc(
        &self,
        request: Request<FencePersonRequest>,
    ) -> Result<Response<FencePersonResponse>, Status> {
        let partition = partition_from_metadata(&request)?;
        let req = request.into_inner();
        self.validate_partition(partition, req.team_id, req.person_id)?;
        // Ownership below is in-process state, which is what a lease
        // lapse makes stale: a fence installed here gates no writes.
        self.check_authority(partition)?;
        let op_id = Uuid::parse_str(&req.op_id)
            .map_err(|_| Status::invalid_argument("op_id must be a valid UUID"))?;
        let op_type = req.op_type();
        if op_type == LifecycleOpType::Unspecified {
            return Err(Status::invalid_argument("op_type must be specified"));
        }
        // A fence installed anywhere but the current owner gates nothing:
        // ownership covers a pod that already handed the partition off, the
        // inflight guard covers the drain window; the saga's retry re-routes.
        self.validate_ownership(partition)?;
        let Some(_inflight_guard) = self.inflight.try_begin(partition) else {
            return Err(Status::failed_precondition(format!(
                "partition {partition} is fenced for handoff; writes are rejected"
            )));
        };

        let cache_key = PersonCacheKey {
            team_id: req.team_id,
            person_id: req.person_id,
        };
        let sealed = self.fence_one(partition, cache_key, op_id, op_type).await?;

        self.authoritative_ok(
            partition,
            FencePersonResponse {
                sealed: Some(sealed),
            },
        )
    }

    /// A member off the routed partition is a final answer, not a routing bounce:
    /// the caller regroups and retries each member alone.
    #[allow(clippy::result_large_err)]
    fn validate_batch_member(
        &self,
        partition: u32,
        team_id: i64,
        person_id: i64,
    ) -> Result<(), Status> {
        self.validate_partition(partition, team_id, person_id)
            .map_err(|status| Status::invalid_argument(status.message()))
    }

    pub(super) async fn fence_persons_rpc(
        &self,
        request: Request<FencePersonsRequest>,
    ) -> Result<Response<FencePersonsResponse>, Status> {
        let partition = partition_from_metadata(&request)?;
        let req = request.into_inner();
        if req.person_ids.len() > MAX_LIFECYCLE_BATCH_SIZE {
            return Err(Status::invalid_argument(format!(
                "FencePersons carries {} persons; the cap is {MAX_LIFECYCLE_BATCH_SIZE}",
                req.person_ids.len()
            )));
        }
        let team_id = req.team_id;
        // Ownership first: an unserved partition is a routing bounce, not a bad batch.
        self.validate_ownership(partition)?;
        // The caller grouped its batch by partition and routed it by one
        // member, so every member must hash to the routed partition; a
        // stale partition count refuses the whole batch before any fence.
        for person_id in &req.person_ids {
            self.validate_batch_member(partition, team_id, *person_id)?;
        }
        self.check_authority(partition)?;
        let op_id = Uuid::parse_str(&req.op_id)
            .map_err(|_| Status::invalid_argument("op_id must be a valid UUID"))?;
        let op_type = req.op_type();
        if op_type == LifecycleOpType::Unspecified {
            return Err(Status::invalid_argument("op_type must be specified"));
        }
        let Some(_inflight_guard) = self.inflight.try_begin(partition) else {
            return Err(Status::failed_precondition(format!(
                "partition {partition} is fenced for handoff; writes are rejected"
            )));
        };
        histogram!("personhog_leader_fence_batch_size").record(req.person_ids.len() as f64);

        // Every fence runs to completion before the batch answers, so a
        // sibling's refusal cancels no load in flight under a lock; the
        // bound keeps the fallback pool's queue and the drain hold shallow.
        let fence_futures = req.person_ids.into_iter().map(|person_id| async move {
            let cache_key = PersonCacheKey { team_id, person_id };
            (
                person_id,
                self.fence_one(partition, cache_key, op_id, op_type).await,
            )
        });
        let results: Vec<_> = futures::stream::iter(fence_futures)
            .buffer_unordered(LIFECYCLE_BATCH_CONCURRENCY)
            .collect()
            .await;

        let mut sealed = Vec::with_capacity(results.len());
        let mut not_found = Vec::new();
        let mut failures = Vec::new();
        for (person_id, result) in results {
            match result {
                Ok(person) => sealed.push(FencedPersonSeal {
                    person_id,
                    version: person.version,
                    created_at: person.created_at,
                }),
                Err(status) if status.code() == tonic::Code::NotFound => not_found.push(person_id),
                Err(status) => failures.push(status),
            }
        }
        first_batch_failure(failures)?;

        self.authoritative_ok(partition, FencePersonsResponse { sealed, not_found })
    }

    pub(super) async fn release_fence_rpc(
        &self,
        request: Request<ReleaseFenceRequest>,
    ) -> Result<Response<ReleaseFenceResponse>, Status> {
        let partition = partition_from_metadata(&request)?;
        let req = request.into_inner();
        self.validate_partition(partition, req.team_id, req.person_id)?;
        // The aborted arm would otherwise ack under a lapsed lease; the
        // committed arm's produce and durable-fact acks need no cover.
        self.check_authority(partition)?;
        let op_id = Uuid::parse_str(&req.op_id)
            .map_err(|_| Status::invalid_argument("op_id must be a valid UUID"))?;

        match req.outcome() {
            ReleaseOutcome::Committed => {
                let release = CommittedRelease::validate(
                    req.person_id,
                    req.person_uuid,
                    req.sealed_version,
                    req.created_at,
                    req.tombstone_version,
                )?;
                // Producing to the changelog must respect the handoff
                // write freeze like any write.
                let Some(_inflight_guard) = self.inflight.try_begin(partition) else {
                    return Err(Status::failed_precondition(format!(
                        "partition {partition} is fenced for handoff; writes are rejected"
                    )));
                };
                let lifecycle_db = self.lifecycle_db()?;
                let snapshot = self.mark_snapshot(op_id).await?;
                self.release_committed(
                    partition,
                    req.team_id,
                    op_id,
                    &release,
                    &snapshot,
                    lifecycle_db,
                )
                .await?;
            }
            ReleaseOutcome::Aborted => {
                self.release_aborted(partition, req.team_id, req.person_id, op_id)
                    .await?;
            }
            ReleaseOutcome::Unspecified => {
                return Err(Status::invalid_argument("outcome must be specified"));
            }
        }

        self.authoritative_ok(partition, ReleaseFenceResponse {})
    }

    pub(super) async fn release_fences_rpc(
        &self,
        request: Request<ReleaseFencesRequest>,
    ) -> Result<Response<ReleaseFencesResponse>, Status> {
        let partition = partition_from_metadata(&request)?;
        let req = request.into_inner();
        if req.persons.len() > MAX_LIFECYCLE_BATCH_SIZE {
            return Err(Status::invalid_argument(format!(
                "ReleaseFences carries {} persons; the cap is {MAX_LIFECYCLE_BATCH_SIZE}",
                req.persons.len()
            )));
        }
        let team_id = req.team_id;
        self.validate_ownership(partition)?;
        // Same strict routing check as a batched fence: every member hashes
        // to the routed partition or the whole batch is refused untouched.
        for person in &req.persons {
            self.validate_batch_member(partition, team_id, person.person_id)?;
        }
        // The aborted arm would otherwise ack under a lapsed lease; the
        // committed arm's produce and durable-fact acks need no cover.
        self.check_authority(partition)?;
        let op_id = Uuid::parse_str(&req.op_id)
            .map_err(|_| Status::invalid_argument("op_id must be a valid UUID"))?;
        histogram!("personhog_leader_release_batch_size").record(req.persons.len() as f64);

        match req.outcome() {
            ReleaseOutcome::Committed => {
                let releases = req
                    .persons
                    .into_iter()
                    .map(|person| {
                        CommittedRelease::validate(
                            person.person_id,
                            person.person_uuid,
                            person.sealed_version,
                            person.created_at,
                            person.tombstone_version,
                        )
                    })
                    .collect::<Result<Vec<_>, Status>>()?;
                if releases.is_empty() {
                    return self.authoritative_ok(partition, ReleaseFencesResponse {});
                }
                // Producing to the changelog must respect the handoff
                // write freeze like any write.
                let Some(_inflight_guard) = self.inflight.try_begin(partition) else {
                    return Err(Status::failed_precondition(format!(
                        "partition {partition} is fenced for handoff; writes are rejected"
                    )));
                };
                let lifecycle_db = self.lifecycle_db()?;
                let snapshot = self.mark_snapshot(op_id).await?;
                // The releases run together so their death documents share
                // fencing windows, and every one runs to completion: a
                // sibling's failure cancels no produce in flight.
                let release_futures: Vec<_> = releases
                    .iter()
                    .map(|release| {
                        self.release_committed(
                            partition,
                            team_id,
                            op_id,
                            release,
                            &snapshot,
                            lifecycle_db,
                        )
                    })
                    .collect();
                let results: Vec<_> = futures::stream::iter(release_futures)
                    .buffer_unordered(LIFECYCLE_BATCH_CONCURRENCY)
                    .collect()
                    .await;
                first_batch_failure(results.into_iter().filter_map(Result::err).collect())?;
            }
            ReleaseOutcome::Aborted => {
                let mut failures = Vec::new();
                for person in &req.persons {
                    if let Err(status) = self
                        .release_aborted(partition, team_id, person.person_id, op_id)
                        .await
                    {
                        failures.push(status);
                    }
                }
                first_batch_failure(failures)?;
            }
            ReleaseOutcome::Unspecified => {
                return Err(Status::invalid_argument("outcome must be specified"));
            }
        }

        self.authoritative_ok(partition, ReleaseFencesResponse {})
    }
}