Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

101 KB
Raw Blame
use std::collections::{HashMap, HashSet};
use std::result::Result as StdResult;
use std::str::from_utf8;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use std::time::{Duration, Instant};

use async_trait::async_trait;
use etcd_client::{EventType, WatchStream};
use futures::future::BoxFuture;
use futures::stream::{FuturesUnordered, StreamExt};
use metrics::{counter, gauge, histogram};
use tokio::sync::{Notify, Semaphore, SemaphorePermit};
use tokio::task::JoinError;
use tokio_util::sync::CancellationToken;

use assignment_coordination::store::parse_watch_value;
use k8s_awareness::types::{ControllerKind, ControllerRef};
use k8s_awareness::{DepartureReason, K8sAwareness};

use crate::authority::AuthorityClock;
use crate::error::{Error, Result};
use crate::store::{self, PersonhogStore};
use crate::types::ChangelogConfig;
use crate::types::{
    HandoffPhase, HandoffState, PartitionAssignment, PodDrainedAck, PodStatus, PodWarmedAck,
    RegisteredPod,
};
use crate::util;

/// The state this pod should hold for one partition, derived purely from
/// the durable coordination state — the partition's assignment and any
/// in-flight handoff. `PodHandle::converge` drives local state to match,
/// so a pod whose memory has diverged from etcd (most notably after a
/// crash-restart inside its lease TTL, which preserves its registration
/// and assignments but wipes its cache and fences) is repaired by
/// re-deriving rather than by replaying remembered events.
/// Public because the stateright model (`personhog-stateright`) drives
/// its pod transitions through this exact function — the model checks
/// the code production runs, so the pod state machine cannot drift from
/// its verified form.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DesiredState {
    /// This pod owns the partition and no handoff constrains it: cache
    /// warm, writes admitted. Also the old owner's state during Freezing
    /// (routers are still collecting the freeze quorum; writes keep
    /// flowing until every router has stashed).
    Serving,
    /// This pod is the old owner of a handoff past the freeze quorum:
    /// writes fenced, inflight drained. `ack` is true while the protocol
    /// is waiting on this pod's DrainedAck (phase Draining) and false once
    /// the ack has been consumed (phase Warming) — re-acking after the
    /// phase advanced would risk orphaning an ack past the coordinator's
    /// cleanup. The cache is deliberately not warmed here: a cold restart
    /// mid-handoff serves read errors until cutover (or until a
    /// cancellation converges back to Serving, which warms).
    Drained { ack: bool },
    /// This pod is the new owner of a handoff in Warming: populate the
    /// cache from Kafka and write a WarmedAck.
    Acquiring,
    /// This pod must not hold the partition: it isn't assigned to it, its
    /// outbound handoff completed, or it is a new owner whose handoff
    /// hasn't reached Warming (warming early would snapshot an HWM the old
    /// owner is still advancing).
    Released,
}

/// Whether `pod` is the incoming owner of a handoff still freezing or draining.
/// A hold's connections are the leader's seal duty to prepare, not this hint's.
pub fn prepare_hinted(pod: &str, handoff: Option<&HandoffState>) -> bool {
    handoff.is_some_and(|h| {
        h.new_owner == pod
            && !h.is_held()
            && matches!(h.phase, HandoffPhase::Freezing | HandoffPhase::Draining)
    })
}

/// Pure derivation of a pod's desired state for a partition. The handoff,
/// when it involves this pod, takes precedence over the assignment: the
/// assignment names the *old* owner (or nobody) for the whole life of a
/// handoff and only flips to the new owner atomically at Complete.
pub fn desired_state(
    pod: &str,
    assignment: Option<&PartitionAssignment>,
    handoff: Option<&HandoffState>,
) -> DesiredState {
    if let Some(h) = handoff {
        // A migration restart: created at Warming, it re-acquires over the new changelog.
        if h.old_owner.as_deref() == Some(pod) && h.new_owner == pod && !h.is_held() {
            match h.phase {
                HandoffPhase::Warming => return DesiredState::Acquiring,
                HandoffPhase::Complete => return DesiredState::Serving,
                HandoffPhase::Freezing | HandoffPhase::Draining => {}
            }
        }
        if h.old_owner.as_deref() == Some(pod) {
            return match h.phase {
                HandoffPhase::Freezing => DesiredState::Serving,
                HandoffPhase::Draining => DesiredState::Drained { ack: true },
                HandoffPhase::Warming => DesiredState::Drained { ack: false },
                HandoffPhase::Complete => DesiredState::Released,
            };
        }
        if h.new_owner == pod {
            return match h.phase {
                HandoffPhase::Freezing | HandoffPhase::Draining => DesiredState::Released,
                HandoffPhase::Warming => DesiredState::Acquiring,
                // The assignment flips to this pod in the same txn that
                // writes Complete.
                HandoffPhase::Complete => DesiredState::Serving,
            };
        }
        // A handoff between two other pods constrains nothing for this
        // one; fall through to the assignment.
    }
    match assignment {
        Some(a) if a.owner == pod => DesiredState::Serving,
        _ => DesiredState::Released,
    }
}

/// Trait for the application-layer handoff handler on writer pods.
///
/// Implementations do the actual work of draining, warming, and releasing
/// partition ownership. Called by `PodHandle` in response to handoff phase
/// transitions it observes via etcd.
#[async_trait]
pub trait HandoffHandler: Send + Sync {
    /// Old owner: wait for all inflight request handlers for this partition to
    /// complete. Because the produce path awaits delivery before returning, this
    /// implies every write ever acked by this pod is durably in Kafka.
    ///
    /// Called when this pod is `old_owner` and handoff phase reaches `Freezing`.
    async fn drain_partition_inflight(&self, partition: u32) -> Result<()>;

    /// New owner: populate the cache from Kafka up to current HWM.
    ///
    /// Called when this pod is `new_owner` and handoff phase reaches `Warming`.
    /// The HWM is guaranteed stable at this point — the old owner has drained
    /// and no router is producing for this partition.
    ///
    /// May be cancelled mid-flight (the coordination loop is dropped on
    /// lease loss and shutdown) and re-invoked later for the same
    /// partition. Implementations must therefore leave no observable
    /// partial state on cancellation — buffer and install atomically, as
    /// the leader's warm does — because a cancelled warm is never
    /// released: the pod only tracks a warm once this returns.
    async fn warm_partition(&self, partition: u32) -> Result<()>;

    /// Owner: confirm this pod still holds whatever a partition needs in
    /// order to serve it, and re-take anything missing.
    ///
    /// Called on every convergence to `Serving`, including reconcile
    /// ticks, so it is the repair path for state the handoff protocol has
    /// no way back from — the leader uses it to re-take a changelog fence
    /// evicted by a broker rejection or lost to a failed abort. Running
    /// under `Serving` is what makes it safe: the pod re-takes only what
    /// the durable assignment says it owns, rather than what its local
    /// caches happen to still hold.
    ///
    /// Idempotent and cheap when nothing is missing, since it runs
    /// per-partition on every tick.
    /// Returns whether repair work was applied, so the convergence can
    /// count it as progress.
    async fn verify_serving(&self, _partition: u32) -> Result<bool> {
        Ok(false)
    }

    /// The handoff names this pod as the incoming owner, but acquisition
    /// is not yet permitted (the old owner is still freezing or
    /// draining). A hint, not a phase: implementations may use the
    /// window to prepare state whose setup touches nothing shared — the
    /// leader pre-connects its changelog producer here so the fence
    /// acquisition inside the warm pays only the init round trip. Called
    /// on every convergence observing that window, so implementations
    /// must be idempotent and must not block: spawn and return.
    async fn prepare_acquire(&self, _partition: u32) {}

    /// The config a strong read captured just before the `warm_partition` that follows.
    async fn capture_changelog(&self, _partition: u32, _config: &ChangelogConfig) {}

    /// Old owner: release the partition from this pod's local state (drop cache,
    /// close consumers, etc.).
    ///
    /// Called when this pod is `old_owner` and handoff phase reaches `Complete`.
    async fn release_partition(&self, partition: u32) -> Result<()>;

    /// The handoff for a partition this pod still owns was cancelled before
    /// completing (e.g. the new owner died mid-warm and the coordinator
    /// deleted the record). The pod remains the owner and must resume
    /// normal service — in particular, re-admit any writes it fenced when
    /// it drained.
    ///
    /// Called when a handoff record is deleted while this pod still holds
    /// the partition (a delete after `Complete` is normal cleanup and does
    /// not trigger this).
    async fn resume_partition(&self, partition: u32) -> Result<()>;
}

#[derive(Debug, Clone)]
pub struct PodConfig {
    pub pod_name: String,
    /// Pod-template-hash (Deployment) or controller-revision-hash (StatefulSet).
    /// Populated via K8s awareness before registration. Empty when K8s awareness is disabled.
    pub generation: String,
    /// The K8s controller (Deployment/StatefulSet) that owns this pod.
    /// Populated via K8s awareness before registration. None when K8s awareness is disabled.
    pub controller: Option<ControllerRef>,
    pub lease_ttl: i64,
    pub heartbeat_interval: Duration,
    /// How long to wait for partitions to drain before shutting down.
    /// Should be less than K8s terminationGracePeriodSeconds to allow
    /// time for lease revocation before SIGKILL.
    pub drain_timeout: Duration,
    /// How often the watch loop re-derives the involved-partition set
    /// from a fresh snapshot, independent of events — the same truth
    /// path the router's reconcile pass provides. Runs as an arm of the
    /// watch loop; each pass re-dispatches every involved partition
    /// through the same single-flight convergence lanes the event path
    /// uses.
    pub reconcile_interval: Duration,
    /// How many consecutive reconcile failures to tolerate before
    /// failing the run. Same reasoning as the router's budget: a failed
    /// pass only means staleness equal to one tick, brief etcd blips
    /// must not restart the fleet, and sustained outage self-fences
    /// through the lease keepalive.
    pub reconcile_failure_budget: u32,
    /// How many consecutive coordination-attempt failures the run
    /// supervisor tolerates before giving up and letting the process
    /// restart. An attempt that made real progress resets the count.
    pub run_retry_budget: u32,
    /// Base backoff between coordination attempts; doubles per
    /// consecutive failure up to a fixed cap.
    pub run_retry_backoff: Duration,
    /// `host:port` where this pod's gRPC server is reachable; registered
    /// so routers can dial the pod through the routing table.
    pub advertise_address: Option<String>,
    /// Maximum number of partition warms this pod runs concurrently.
    /// Warming several partitions at once keeps a deploy-burst of
    /// inbound handoffs from queueing each warm behind the last; the
    /// bound keeps a cold restart from opening one Kafka replay per
    /// assigned partition all at once. Only warms queue on this —
    /// drains, releases, and acks are never held behind warm pressure.
    pub warm_concurrency: usize,
}

/// How long either exit path waits for its lease revoke before moving
/// on: unbounded, it waits out the very etcd outage that usually runs
/// it, while the registration expires by TTL anyway. Public because the
/// leader binary validates its shutdown budget against it at startup.
pub const REVOKE_TIMEOUT: Duration = Duration::from_secs(5);

/// The fence's drain bound on the shutdown path: its value there is
/// stopping admissions and a short straggler grace — anything the
/// graceful drain ahead of it could not quiesce will not quiesce now.
/// Public because the leader binary sums it into its budget check.
pub const SHUTDOWN_FENCE_BOUND: Duration = Duration::from_secs(3);

/// How long the drain's bookkeeping (Draining write, involvement
/// snapshot, watch creation) may take before degrading to the
/// fence-and-revoke teardown. These run before the drain's own timeout
/// starts, and an etcd whose KV path stalls while its lease path stays
/// healthy would otherwise hold shutdown here indefinitely. Public
/// because the leader binary sums it into its budget check.
pub const DRAIN_SETUP_BOUND: Duration = Duration::from_secs(5);

impl Default for PodConfig {
    fn default() -> Self {
        Self {
            pod_name: "writer-0".to_string(),
            generation: String::new(),
            controller: None,
            lease_ttl: 30,
            heartbeat_interval: Duration::from_secs(10),
            drain_timeout: Duration::from_secs(30),
            reconcile_interval: Duration::from_secs(5),
            reconcile_failure_budget: 12,
            run_retry_budget: 10,
            run_retry_backoff: Duration::from_millis(500),
            advertise_address: None,
            warm_concurrency: 4,
        }
    }
}

/// What a partition's warm was installed for. A warm is only as fresh as
/// the moment it replayed the changelog; `converge` records why each one
/// was installed so a later Acquiring can tell a warm it may honor from
/// one left over from an earlier era.
#[derive(Debug, Clone, PartialEq, Eq)]
enum WarmProvenance {
    /// Warmed as (or on the way back to being) the serving owner: a
    /// process restart, or the returning old owner of an in-flight
    /// handoff. Current at install time and kept current by the pod's
    /// own accepted writes for as long as it remains the owner.
    Serving,
    /// Warmed for one specific handoff's Warming phase, identified by
    /// its id.
    Handoff(String),
}

pub struct PodHandle {
    store: Arc<PersonhogStore>,
    config: PodConfig,
    handler: Arc<dyn HandoffHandler>,
    /// Partitions warmed by this process — local, dies with the process
    /// — each with the provenance of its warm.
    ///
    /// A std mutex on purpose, for both maps: an async lock leaves a
    /// suspension point between the warm installing in the data plane
    /// and the insert recording it, and a lane dropped there leaves a
    /// warm the self-fence cannot see. Guards held across an await are
    /// a compile error (`run` is spawned, the guard is not `Send`) and
    /// a clippy lint besides.
    warmed_partitions: StdMutex<HashMap<u32, WarmProvenance>>,
    /// Partitions this process has write-fenced via a drain — local,
    /// consulted so convergence to Serving only issues a resume when a
    /// fence actually exists. See `warmed_partitions` for why the lock
    /// is synchronous.
    fenced_partitions: StdMutex<HashSet<u32>>,
    /// Signalled when a partition is released, waking `drain()` without polling.
    drain_notify: Notify,
    /// Bounds concurrent `warm_partition` calls to `warm_concurrency`.
    warm_slots: Semaphore,
    /// Set when a lease-loss self-fence failed: local serving state may
    /// not reflect lost ownership, so the run supervisor must not retry
    /// in place — only a process restart clears this.
    fence_poisoned: AtomicBool,
    /// This pod's claim to serve, shared with the data plane and reset
    /// at each lease grant. Reads as invalid until the first grant.
    authority: Arc<AuthorityClock>,
    /// Optional K8s awareness for departure classification during shutdown.
    k8s_awareness: Option<Arc<K8sAwareness>>,
    /// Nudged when serving state broke in a way only a convergence can
    /// mend (the leader nudges when a changelog producer is condemned).
    /// The watch loop answers with an early reconcile pass, so repair
    /// happens now rather than on the next tick.
    repair_nudge: Option<Arc<Notify>>,
}

impl PodHandle {
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        store: Arc<PersonhogStore>,
        config: PodConfig,
        handler: Arc<dyn HandoffHandler>,
        k8s_awareness: Option<Arc<K8sAwareness>>,
        // Shared with the data plane, which holds it from process start
        // and consults it per request.
        authority: Arc<AuthorityClock>,
    ) -> Self {
        let renewal_margin = AuthorityClock::renewal_margin(config.lease_ttl);
        assert!(
            config.heartbeat_interval < renewal_margin,
            "heartbeat_interval ({:?}) must be well under the keepalive renewal margin \
             (2/3 of lease_ttl = {renewal_margin:?}): the post-renewal sleep alone would \
             exhaust the margin and the pod would fence-loop against healthy etcd",
            config.heartbeat_interval,
        );
        let warm_slots = Semaphore::new(config.warm_concurrency);
        Self {
            store,
            config,
            handler,
            warmed_partitions: StdMutex::new(HashMap::new()),
            fenced_partitions: StdMutex::new(HashSet::new()),
            drain_notify: Notify::new(),
            warm_slots,
            fence_poisoned: AtomicBool::new(false),
            authority,
            k8s_awareness,
            repair_nudge: None,
        }
    }

    /// A strong read, never the adopted copy: a lagging watch would warm from a retired topic.
    async fn capture_changelog(
        &self,
        partition: u32,
        required_epoch: u64,
    ) -> Result<Option<ChangelogConfig>> {
        let Some(config) = self.store.get_changelog_config().await? else {
            counter!("personhog_coordination_acquisitions_parked_total").increment(1);
            tracing::warn!(
                partition,
                "config/changelog is not seeded; parking the acquisition"
            );
            return Ok(None);
        };
        if config.epoch < required_epoch {
            counter!("personhog_coordination_acquisitions_parked_total").increment(1);
            tracing::warn!(
                partition,
                read_epoch = config.epoch,
                required_epoch,
                "config/changelog is older than the handoff's epoch; parking the acquisition"
            );
            return Ok(None);
        }
        self.handler.capture_changelog(partition, &config).await;
        Ok(Some(config))
    }

    /// Release the partition from the handler and forget its warm here.
    async fn release_warm(&self, partition: u32) -> Result<()> {
        self.handler.release_partition(partition).await?;
        self.warmed_partitions
            .lock()
            .expect("warmed partitions lock poisoned")
            .remove(&partition);
        Ok(())
    }

    /// Run a reconcile pass whenever `nudge` fires, in addition to the
    /// periodic tick. The nudging end announces breakage the protocol
    /// has no event for — the leader's condemned changelog producer —
    /// and this is what turns its repair latency from one reconcile
    /// interval into one convergence.
    pub fn with_repair_nudge(mut self, nudge: Arc<Notify>) -> Self {
        self.repair_nudge = Some(nudge);
        self
    }

    /// This pod's claim to serve, for the data plane to consult on the
    /// request path. Invalid until the first lease is granted.
    pub fn authority(&self) -> Arc<AuthorityClock> {
        Arc::clone(&self.authority)
    }

    /// Run the pod's coordination, supervised at two levels so a
    /// failure costs exactly the authority it invalidates and nothing
    /// more. The **session** level owns the lease: grant, register,
    /// heartbeat. The **attempt** level (`run_once`) owns only
    /// convergence and the handoff watch.
    ///
    /// An attempt failure — a broken watch stream, a failed etcd write,
    /// a phase-handler error — retries in place with the lease and
    /// registration intact: the coordinator keeps seeing a live pod, so
    /// it never runs the dead-owner path against a process that is
    /// still serving, and the data plane keeps its cache. That is the
    /// invariant this split enforces: serving authority is held exactly
    /// as long as the lease-backed registration the coordinator's view
    /// derives from. A handoff that tries to move a partition away
    /// while the watch is down stalls awaiting this pod's ack and
    /// resolves through the phase-deadline replacement machinery.
    ///
    /// Only lease loss ends a session, and it still self-fences — now
    /// locally: every held partition is released before a new session
    /// registers afresh, because the coordinator treats the expired
    /// lease as death and may already be reassigning. Re-acquisition
    /// always re-warms. A failed local fence poisons recovery and the
    /// error propagates so a process restart clears everything.
    ///
    /// The registration asymmetry with the router is deliberate: a pod
    /// holds data authority, so its registration must survive attempt
    /// failures; a router holds freeze-quorum membership, so it sheds
    /// its registration fast — a registered but non-acking router would
    /// stall every freeze quorum until the phase deadline.
    pub async fn run(&self, cancel: CancellationToken) -> Result<()> {
        util::preregister_pod_metrics();
        let mut consecutive_failures = util::RunFailures::default();
        // Set by the coordination loop whenever it applies real work
        // (a convergence completed); consumed by each failure note to
        // decide crash-loop vs fresh failure.
        let progress = AtomicBool::new(false);
        loop {
            // Cancel-aware internally — see `begin_session` for why the
            // race lives inside the call rather than around it.
            let (lease_id, granted_at) = match self.begin_session(&cancel).await {
                Ok(None) => return Ok(()),
                Ok(Some(session)) => session,
                Err(e) => {
                    if cancel.is_cancelled() {
                        return Ok(());
                    }
                    if !self.note_run_failure(&mut consecutive_failures, &progress, &e) {
                        return Err(e);
                    }
                    if self
                        .run_backoff(&cancel, consecutive_failures.consecutive())
                        .await
                    {
                        return Ok(());
                    }
                    continue;
                }
            };

            // The heartbeat is session-scoped and outlives attempts: were
            // it attempt-scoped, a failed attempt would let the lease
            // lapse and reopen the deregistered-but-serving hole the
            // session split exists to close. It keeps running through the
            // drain phase too, so the coordinator sees a Draining pod
            // rather than a crashed one.
            // A new lease is a new claim: reset the shared clock rather
            // than replacing it, so the data plane's handle carries the
            // new session without re-plumbing.
            self.authority.begin_session(
                AuthorityClock::renewal_margin(self.config.lease_ttl),
                granted_at,
            );

            // The keepalive learns of a revoked lease on its next round,
            // which is up to a heartbeat away — and in that window the
            // coordinator has already seen the deletion and can reassign,
            // warm a successor, and let it start accepting writes while
            // this pod still answers reads from a cache that is no longer
            // the truth. Watching our own registration collapses that
            // window to a watch delivery.
            //
            // Deliberately best-effort: it accelerates detection, it does
            // not own it. If the stream never establishes or dies, the
            // keepalive's margin remains the guarantee, exactly as before.
            let heartbeat_cancel = CancellationToken::new();
            let registration_cancel = CancellationToken::new();
            let registration_watch = {
                let store = Arc::clone(&self.store);
                let authority = Arc::clone(&self.authority);
                let pod_name = self.config.pod_name.clone();
                let token = registration_cancel.child_token();
                // Ending the session is the keepalive's job, so the watch
                // ends it the same way rather than inventing a second
                // path: stopping the heartbeat makes the attempt loop
                // take the lease-loss branch it already has, which
                // fences, releases, and registers anew.
                let end_session = heartbeat_cancel.clone();
                tokio::spawn(async move {
                    watch_own_registration(store, pod_name, authority, end_session, token).await;
                })
            };

            let mut heartbeat_handle = {
                let store = Arc::clone(&self.store);
                let interval = self.config.heartbeat_interval;
                let lease_ttl = self.config.lease_ttl;
                let token = heartbeat_cancel.child_token();
                let authority = Arc::clone(&self.authority);
                tokio::spawn(async move {
                    util::run_lease_keepalive(
                        store,
                        lease_id,
                        interval,
                        lease_ttl,
                        granted_at,
                        "pod",
                        Some(authority),
                        token,
                    )
                    .await
                })
            };

            let mut lease_err: Option<Error> = None;
            let mut fatal: Option<Error> = None;
            // Attempts under this lease. The outer `select!` is what
            // guarantees prompt exit even when the watch loop is parked
            // inside a phase handler: the loop's own cancel check only
            // runs between iterations, so racing the token here drops the
            // in-flight future via cancel-by-drop, unwinding a stuck
            // handler. The heartbeat is raced for the same reason — a pod
            // that outlives its lease is a zombie, and lease loss must
            // end the session immediately.
            loop {
                let result = tokio::select! {
                    r = self.run_once(cancel.clone(), &progress) => r,
                    r = &mut heartbeat_handle => {
                        let err = Self::heartbeat_exit_error(r);
                        tracing::error!(
                            pod = %self.config.pod_name,
                            error = %err,
                            "lease keepalive failed; self-fencing"
                        );
                        lease_err = Some(err);
                        break;
                    }
                    _ = cancel.cancelled() => Ok(()),
                };
                if cancel.is_cancelled() {
                    break;
                }
                let err = match result {
                    Ok(()) => break,
                    Err(e) => e,
                };
                if !self.note_run_failure(&mut consecutive_failures, &progress, &err) {
                    fatal = Some(err);
                    break;
                }
                // The nap races the heartbeat: every await under a live
                // lease must observe lease loss promptly, and this one
                // otherwise defers it by a full backoff period — the
                // coordinator may already be reassigning while the pod
                // serves, unaware, until the nap ends.
                tokio::select! {
                    stop = self.run_backoff(&cancel, consecutive_failures.consecutive()) => {
                        if stop {
                            break;
                        }
                    }
                    r = &mut heartbeat_handle => {
                        let err = Self::heartbeat_exit_error(r);
                        tracing::error!(
                            pod = %self.config.pod_name,
                            error = %err,
                            "lease keepalive failed during backoff; self-fencing"
                        );
                        lease_err = Some(err);
                        break;
                    }
                }
                // Next attempt runs under the same lease: the
                // registration was never given up, so the coordinator's
                // view of this pod is unbroken.
            }

            // Graceful drain only on external shutdown with a live
            // lease. Skipped on lease loss: the coordinator already
            // considers this pod dead and is reassigning via the
            // dead-owner path — a graceful drain would only race it, and
            // every status write would fail against the expired lease.
            // The drain itself races the heartbeat for the same reason
            // as every other live-lease await: the pod serves its
            // partitions until their handoffs complete, so a lease lost
            // mid-drain must switch to the local self-fence immediately
            // rather than draining leaseless until the timeout.
            if cancel.is_cancelled() && lease_err.is_none() {
                tokio::select! {
                    r = self.drain(lease_id, &progress) => {
                        if let Err(e) = r {
                            tracing::warn!(pod = %self.config.pod_name, error = %e, "drain failed");
                        }
                    }
                    r = &mut heartbeat_handle => {
                        let err = Self::heartbeat_exit_error(r);
                        tracing::error!(
                            pod = %self.config.pod_name,
                            error = %err,
                            "lease keepalive failed during drain; self-fencing"
                        );
                        lease_err = Some(err);
                    }
                }
            }
            let lease_lost = lease_err.is_some();

            if !lease_lost {
                // Fence before the registration disappears: revoking the
                // lease deletes it instantly, and the dead-owner path the
                // coordinator then runs has no fence of its own — its
                // safety rests on a deregistered owner being unable to
                // produce. This matters most on the budget-exhausted
                // path, where the pod still holds and serves every
                // partition; after a completed graceful drain it is a
                // no-op. The heartbeat is still running here, so the
                // fence has no lease-runway pressure — but on the
                // shutdown path it does have a lifecycle budget: the
                // graceful drain may already have spent its full
                // timeout, and the process supervisor's window and the
                // pod's termination grace period are sized around one
                // drain timeout plus headroom, not two. Anything the
                // drain could not quiesce in thirty seconds will not
                // quiesce now; the fence's value here is stopping
                // admissions (milliseconds) and giving stragglers a
                // short grace, so its drain bound is a few seconds. The
                // budget-exhausted path is not on the shutdown clock and
                // keeps the full timeout.
                let fence_bound = if cancel.is_cancelled() {
                    SHUTDOWN_FENCE_BOUND.min(self.config.drain_timeout)
                } else {
                    self.config.drain_timeout
                };
                if let Err(e) = self.self_fence_locally(fence_bound).await {
                    self.fence_poisoned.store(true, Ordering::SeqCst);
                    tracing::error!(
                        pod = %self.config.pod_name,
                        error = %e,
                        "pre-revoke self-fence failed; refusing in-place recovery"
                    );
                }
                // Only now do we stop being the owner. On this path the
                // lease is still alive and the registration still stands:
                // surrendering before the drain would refuse reads the
                // protocol deliberately keeps serving — the old owner's
                // cache is the latest state right up to cutover — while
                // the coordinator, seeing a live owner, reassigns
                // nothing.
                self.authority.surrender();
                registration_cancel.cancel();
                drop(registration_watch.await);
                heartbeat_cancel.cancel();
                drop(heartbeat_handle.await);
                // Bounded like the lease-loss path's revoke: this path
                // is on the shutdown clock, and an unbounded revoke
                // against a hung etcd would spend the termination grace
                // the drain and fence are budgeted out of.
                drop(tokio::time::timeout(REVOKE_TIMEOUT, self.store.revoke_lease(lease_id)).await);
            } else {
                // Self-fence locally before any new session: every held
                // partition is released, dropping its cache and serving
                // authority. Re-acquisition always re-warms, so nothing
                // is lost but memory. A failed release poisons recovery.
                // The drain budget is the lease runway the keepalive
                // margin reserved — a third of the TTL — not the full
                // graceful drain timeout: the coordinator reassigns at
                // expiry, and a fence still draining past it loses the
                // race it exists to win. Overshoot poisons, and the
                // process restart clears stragglers by death.
                // Authority is already gone, so stop serving as the owner
                // before the drain rather than after it: the coordinator
                // may be reassigning right now, and a drain can take
                // seconds. A stale stamp merely expires; lease loss is a
                // fact that must not be undone by a renewal racing in
                // behind us, which is why this latches.
                self.authority.surrender();
                registration_cancel.cancel();
                drop(registration_watch.await);
                let runway = Duration::from_secs(self.config.lease_ttl.max(0) as u64) / 3;
                if let Err(e) = self
                    .self_fence_locally(runway.min(self.config.drain_timeout))
                    .await
                {
                    self.fence_poisoned.store(true, Ordering::SeqCst);
                    tracing::error!(
                        pod = %self.config.pod_name,
                        error = %e,
                        "local self-fence failed; refusing in-place recovery"
                    );
                }
                // Usually the lease is already dead here and this is a
                // dropped error. The registration watch also lands on
                // this branch when the key was deleted out from under a
                // live lease (an operator `del`, not a revoke) — without
                // this, that lease would sit alive and unreferenced for
                // its full TTL while the next session grants a second
                // one. Strictly after the self-fence and bounded: this
                // is cleanup, an unhealthy etcd is the usual reason for
                // being on this branch, and the store has no request
                // timeouts of its own — unbounded it could hold up the
                // fence or the next session for as long as the outage.
                drop(tokio::time::timeout(REVOKE_TIMEOUT, self.store.revoke_lease(lease_id)).await);
            }

            if let Some(e) = fatal {
                return Err(e);
            }
            if cancel.is_cancelled() {
                return Ok(());
            }
            match lease_err {
                Some(e) => {
                    if self.fence_poisoned.load(Ordering::SeqCst) {
                        // Serving state may not reflect the lost
                        // ownership; only a process restart clears it.
                        return Err(e);
                    }
                    if !self.note_run_failure(&mut consecutive_failures, &progress, &e) {
                        return Err(e);
                    }
                    if self
                        .run_backoff(&cancel, consecutive_failures.consecutive())
                        .await
                    {
                        return Ok(());
                    }
                    // New session: fresh lease, fresh registration,
                    // ownership re-acquired through the warm path.
                }
                None => return Ok(()),
            }
        }
    }

    /// Classify the session keepalive task's exit. Any exit while the
    /// session is live means the lease can no longer be trusted.
    fn heartbeat_exit_error(result: StdResult<Result<()>, JoinError>) -> Error {
        match result {
            Ok(Ok(())) => Error::invalid_state("lease keepalive exited unexpectedly".to_string()),
            Ok(Err(e)) => e,
            Err(join_err) => Error::invalid_state(format!("keepalive task panicked: {join_err}")),
        }
    }

    /// Grant a fresh lease and register under it — the start of a
    /// coordination session. Returns `None` when cancellation arrived
    /// first.
    ///
    /// The race lives inside because the two steps abandon differently:
    /// an abandoned grant leaves only a lease that expires on its TTL,
    /// but an abandoned registration can still land — a phantom pod the
    /// coordinator plans toward — so past the grant, any abandonment
    /// revokes the lease, which deletes a landed registration or makes
    /// etcd reject a late one. Only a revoke that itself times out
    /// leaves the phantom, TTL-bounded; see the README's residual.
    async fn begin_session(&self, cancel: &CancellationToken) -> Result<Option<(i64, Instant)>> {
        // The server's TTL countdown starts at the grant; anchoring the
        // keepalive's margin clock any later would overstate runway by
        // however long registration took.
        let granted_at = Instant::now();
        let lease_id = tokio::select! {
            _ = cancel.cancelled() => return Ok(None),
            granted = self.store.grant_lease(self.config.lease_ttl) => granted?,
        };
        let registered = tokio::select! {
            _ = cancel.cancelled() => {
                drop(tokio::time::timeout(REVOKE_TIMEOUT, self.store.revoke_lease(lease_id)).await);
                return Ok(None);
            }
            registered = self.register(lease_id) => registered,
        };
        if let Err(e) = registered {
            // A failed registration may also have half-landed; the same
            // revoke clears it rather than leaving the lease to its TTL.
            drop(tokio::time::timeout(REVOKE_TIMEOUT, self.store.revoke_lease(lease_id)).await);
            return Err(e);
        }
        tracing::info!(pod = %self.config.pod_name, "registered with etcd");
        Ok(Some((lease_id, granted_at)))
    }

    /// See `util::note_run_failure` for the progress-based reset
    /// semantics.
    fn note_run_failure(
        &self,
        failures: &mut util::RunFailures,
        progress: &AtomicBool,
        err: &Error,
    ) -> bool {
        util::note_run_failure(
            failures,
            progress,
            self.config.run_retry_budget,
            "pod",
            &self.config.pod_name,
            err,
        )
    }

    /// Exponential backoff between recovery steps. Returns `true` when
    /// cancelled during the wait.
    async fn run_backoff(&self, cancel: &CancellationToken, consecutive: u32) -> bool {
        const BACKOFF_CAP: Duration = Duration::from_secs(15);
        let backoff = self
            .config
            .run_retry_backoff
            .saturating_mul(2u32.saturating_pow(consecutive.saturating_sub(1)))
            .min(BACKOFF_CAP);
        tokio::select! {
            _ = cancel.cancelled() => true,
            _ = tokio::time::sleep(backoff) => false,
        }
    }

    /// One coordination attempt under a live session: converge every
    /// involved partition from a fresh snapshot, then watch. Holds no
    /// lease state — the session and its heartbeat outlive any number of
    /// failed attempts. Convergence-before-watching is what makes each
    /// attempt (and a crash-restart within the lease TTL) safe: local
    /// state is re-derived from durable state, cold assigned partitions
    /// re-warm, in-flight handoffs get their drain/warm/ack, completed
    /// ones release, and the watch anchors to the snapshot revision so
    /// nothing between snapshot and attach is lost.
    async fn run_once(&self, cancel: CancellationToken, progress: &AtomicBool) -> Result<()> {
        let (initial, snapshot_revision) = self.involved_partitions().await?;
        let stream = self
            .store
            .watch_handoffs_from(snapshot_revision + 1)
            .await?;
        self.watch_handoff_loop(stream, cancel, initial, progress)
            .await
    }

    /// Fence, quiesce, and release every locally held partition —
    /// warmed or fenced — through the handler, clearing the pod's local
    /// ownership state. Purely local: no etcd writes, callable with no
    /// lease.
    ///
    /// The order per partition is load-bearing for what release
    /// exposes, not for the writes already in flight: an admitted write
    /// completes its produce and acks whatever this function does —
    /// neither the fence nor a withheld release can stop it. Draining
    /// first (fence, then wait for the inflight counter) means the
    /// release that follows unfences a partition with nothing left in
    /// flight — no fresh admission lands on a leaseless pod, and the
    /// cache is not dropped out from under a handler still using it —
    /// and once this returns cleanly, everything this pod admitted had
    /// acked before any partition was let go. Each drain is bounded by
    /// the drain timeout; a partition that cannot quiesce stays fenced,
    /// held, and unreleased — its in-flight work's fate belongs to the
    /// broker (fenced, or committed below the successor's cutoff; with
    /// fencing off it is the documented unfenced residual) — and the
    /// caller poisons the run so the process restart clears it by
    /// death, exactly as the pre-supervisor design did. The window before the lease loss is
    /// even *detected* (up to one heartbeat tick) remains the
    /// documented zombie residual; this closes only the part the local
    /// fence itself controls.
    async fn self_fence_locally(&self, drain_bound: Duration) -> Result<()> {
        let held: HashSet<u32> = {
            let warmed = self
                .warmed_partitions
                .lock()
                .expect("warmed partitions lock poisoned");
            let fenced = self
                .fenced_partitions
                .lock()
                .expect("fenced partitions lock poisoned");
            warmed
                .keys()
                .copied()
                .chain(fenced.iter().copied())
                .collect()
        };
        // Phase 1: fence and quiesce every partition CONCURRENTLY. Each
        // drain fences as its first action, so running them together
        // stops all admissions within milliseconds of each other and
        // bounds the whole quiesce by a single drain timeout. Draining
        // sequentially would leave later partitions serving — leaseless
        // — while earlier ones wait out their in-flight work, extending
        // the zombie window per held partition.
        let mut drains = tokio::task::JoinSet::new();
        for &partition in &held {
            let handler = Arc::clone(&self.handler);
            let timeout = drain_bound;
            drains.spawn(async move {
                tokio::time::timeout(timeout, handler.drain_partition_inflight(partition))
                    .await
                    .map_err(|_| {
                        Error::invalid_state(format!(
                            "self-fence drain timed out for partition {partition}"
                        ))
                    })??;
                Ok::<u32, Error>(partition)
            });
        }
        // Failures are collected rather than propagated. This runs
        // because the pod has lost the right to serve, so giving the
        // partitions up matters more than reporting why one of them
        // resisted — returning on the first error would leave every
        // other partition still served by a pod with no lease, which is
        // the zombie this function exists to prevent.
        let mut failures: Vec<String> = Vec::new();
        let mut quiesced: HashSet<u32> = HashSet::new();
        while let Some(joined) = drains.join_next().await {
            match joined {
                Ok(Ok(partition)) => {
                    quiesced.insert(partition);
                }
                Ok(Err(e)) => failures.push(e.to_string()),
                Err(e) => failures.push(format!("self-fence drain panicked: {e}")),
            }
        }

        // Phase 2: release each partition that quiesced — dropping its
        // cache and serving authority — and clear the local ownership
        // state for it.
        //
        // Only the ones that quiesced. A write still in flight acks
        // whether or not its partition is released — nothing can stop
        // it — so what releasing an un-quiesced partition would actually
        // do is unfence fresh admissions on a pod with no lease, drop
        // the cache out from under the handlers still using it, and
        // erase the record that the partition was never given up. One
        // left fenced and unreleased stays that way until the process
        // restarts, which is the outcome its drain timing out already
        // implies.
        for partition in held {
            if !quiesced.contains(&partition) {
                continue;
            }
            if let Err(e) = self.release_warm(partition).await {
                failures.push(format!("release of partition {partition}: {e}"));
                // Still held: the handler may retain the cache and the
                // serving authority, so forgetting it here would make the
                // closing gauge read a partition as given up that was
                // not. The run is about to end poisoned either way.
                continue;
            }
            self.fenced_partitions
                .lock()
                .expect("fenced partitions lock poisoned")
                .remove(&partition);
        }
        // Whatever is left, not zero. A partition whose drain never
        // quiesced is still held, and reporting none held would hide it
        // at the one moment the count is worth reading.
        gauge!("personhog_coordination_partitions_held").set(self.held_partition_count() as f64);
        if !failures.is_empty() {
            return Err(Error::invalid_state(format!(
                "self-fence completed with {} failure(s): {}",
                failures.len(),
                failures.join("; ")
            )));
        }
        Ok(())
    }

    async fn register(&self, lease_id: i64) -> Result<()> {
        let now = util::now_seconds();
        let pod = RegisteredPod {
            pod_name: self.config.pod_name.clone(),
            generation: self.config.generation.clone(),
            status: PodStatus::Ready,
            registered_at: now,
            last_heartbeat: now,
            controller: self.config.controller.clone(),
            advertise_address: self.config.advertise_address.clone(),
        };
        self.store.register_pod(&pod, lease_id).await
    }

    /// Classify the departure reason using K8s awareness, if available.
    async fn classify_departure(&self) -> DepartureReason {
        let (Some(k8s), Some(controller)) = (&self.k8s_awareness, &self.config.controller) else {
            return DepartureReason::Unknown;
        };

        k8s.classify_departure(controller, &self.config.generation)
            .await
    }

    /// Graceful drain: set status to Draining, then keep processing handoff
    /// events until all owned partitions have been released or timeout.
    ///
    /// The coordinator sees the Draining status, excludes this pod from
    /// active assignments, and creates handoffs for its partitions. This pod
    /// continues watching for handoff Complete events to release partitions.
    ///
    /// For StatefulSet rollouts, the same pod name comes back with a new
    /// revision, so we skip the drain and exit immediately (ShutdownNow).
    async fn drain(&self, lease_id: i64, progress: &AtomicBool) -> Result<()> {
        let reason = self.classify_departure().await;

        // StatefulSet rollout: same pod name returns, no need to drain
        let is_statefulset_rollout = matches!(
            (&self.config.controller, reason),
            (Some(ref c), DepartureReason::Rollout) if c.kind == ControllerKind::StatefulSet
        );

        if is_statefulset_rollout {
            tracing::info!(
                pod = %self.config.pod_name,
                reason = %reason,
                "StatefulSet rollout detected, shutting down immediately"
            );
            return Ok(());
        }

        // The bookkeeping is bounded as one unit: a timeout surfaces as
        // an error, and a failed drain already degrades to the
        // fence-and-revoke teardown, which is exactly what a shutdown
        // stuck on a stalled KV path should do.
        let setup = async {
            self.store
                .update_pod_status(&self.config.pod_name, PodStatus::Draining, lease_id)
                .await?;

            tracing::info!(
                pod = %self.config.pod_name,
                reason = %reason,
                "set status to Draining, waiting for partition handoffs"
            );

            if self.held_partition_count() == 0 {
                return Ok(None);
            }

            // Keep converging during drain so partitions release as the
            // coordinator completes their handoffs. Reconcile first — a
            // Complete written while the main loop was winding down would
            // otherwise be missed — and anchor the fresh watch to the
            // snapshot's revision.
            let (initial, snapshot_revision) = self.involved_partitions().await?;
            let stream = self
                .store
                .watch_handoffs_from(snapshot_revision + 1)
                .await?;
            Ok::<_, Error>(Some((initial, stream)))
        };
        let Some((initial, stream)) = tokio::time::timeout(DRAIN_SETUP_BOUND, setup)
            .await
            .map_err(|_| {
                Error::invalid_state(format!(
                    "drain setup exceeded {DRAIN_SETUP_BOUND:?}; degrading to fence and revoke"
                ))
            })??
        else {
            tracing::info!(pod = %self.config.pod_name, "no partitions to drain");
            return Ok(());
        };
        let drain_cancel = CancellationToken::new();

        tokio::select! {
            r = self.watch_handoff_loop(stream, drain_cancel.clone(), initial, progress) => {
                r?;
            },
            _ = self.wait_for_drain() => {
                tracing::info!(pod = %self.config.pod_name, "all partitions drained successfully");
            },
            _ = tokio::time::sleep(self.config.drain_timeout) => {
                let remaining = self.held_partition_count();
                tracing::warn!(
                    pod = %self.config.pod_name,
                    remaining_partitions = remaining,
                    "drain timeout exceeded, shutting down"
                );
            }
        }

        drain_cancel.cancel();
        Ok(())
    }

    /// Number of partitions this process still holds state for — warmed
    /// or write-fenced. `drain()` waits for this to reach zero: a fenced
    /// partition's outbound handoff is still in flight, and the pod must
    /// stay alive to release it at Complete.
    fn held_partition_count(&self) -> usize {
        let warmed = self
            .warmed_partitions
            .lock()
            .expect("warmed partitions lock poisoned");
        let fenced = self
            .fenced_partitions
            .lock()
            .expect("fenced partitions lock poisoned");
        warmed
            .keys()
            .chain(fenced.iter())
            .collect::<HashSet<_>>()
            .len()
    }

    /// Wait until all held partitions have been released via handoffs.
    /// Woken reactively by `drain_notify` each time a partition is released.
    async fn wait_for_drain(&self) {
        loop {
            if self.held_partition_count() == 0 {
                return;
            }
            self.drain_notify.notified().await;
        }
    }

    /// Derive every partition this pod is involved in — assigned to it,
    /// named in a handoff as old or new owner, or still held locally —
    /// from a consistent snapshot of the durable state. Returns the set
    /// alongside the smaller of the two snapshot revisions so the caller
    /// can anchor the handoff watch: any change landing between the two
    /// reads (or between them and the watch attaching) is redelivered as
    /// an event and re-converged with fresh reads.
    async fn involved_partitions(&self) -> Result<(HashSet<u32>, i64)> {
        let (assignments, rev_a) = self.store.list_assignments_with_revision().await?;
        let (handoffs, rev_h) = self.store.list_handoffs_with_revision().await?;
        let pod = &self.config.pod_name;

        let mut partitions: HashSet<u32> = HashSet::new();
        for a in &assignments {
            if a.owner == *pod {
                partitions.insert(a.partition);
            }
        }
        for h in &handoffs {
            if h.old_owner.as_deref() == Some(pod.as_str()) || h.new_owner == *pod {
                partitions.insert(h.partition);
            }
        }
        // Locally-held partitions the durable state no longer involves
        // this pod in still need convergence — that is how a warm or
        // fence left over from a departed ownership gets released when
        // the Complete event that should have done it was missed.
        partitions.extend(
            self.warmed_partitions
                .lock()
                .expect("warmed partitions lock poisoned")
                .keys()
                .copied(),
        );
        partitions.extend(
            self.fenced_partitions
                .lock()
                .expect("fenced partitions lock poisoned")
                .iter()
                .copied(),
        );

        tracing::info!(
            pod,
            partitions = partitions.len(),
            "reconciling local state against durable state"
        );

        Ok((partitions, rev_a.min(rev_h)))
    }

    /// Whether this pod still holds something for the partition: a warm
    /// cache or a write fence. Local state outlives the durable record
    /// that created it, which is what makes a pod care about a handoff
    /// that no longer names it — a cancellation leaves the old owner
    /// fenced, and only the fence says so.
    fn holds_local_state(&self, partition: u32) -> bool {
        if self
            .warmed_partitions
            .lock()
            .expect("warmed partitions lock poisoned")
            .contains_key(&partition)
        {
            return true;
        }
        self.fenced_partitions
            .lock()
            .expect("fenced partitions lock poisoned")
            .contains(&partition)
    }
}

/// Whether a handoff event concerns the pod that observed it — the
/// scoping that keeps a fleet-wide rebalance from costing every pod a
/// convergence (two point reads) on every partition's event.
///
/// `handoff` is `None` for a deletion, which carries no owners; only
/// the pod's own involvement can decide, and must — a cancelled handoff
/// has to reach the old owner holding its fence and the new owner still
/// warming. `converging` covers the warm window, where the pod holds
/// neither cache nor fence yet: dispatching coalesces onto the running
/// convergence, which re-derives once the warm completes.
///
/// Split out so both directions can be pinned: widening this to always
/// return true costs nothing any convergence test observes.
fn event_concerns_pod(
    pod_name: &str,
    handoff: Option<&HandoffState>,
    holds_local_state: bool,
    converging: bool,
) -> bool {
    let named = handoff.is_some_and(|h| {
        h.old_owner.as_deref() == Some(pod_name) || h.new_owner.as_str() == pod_name
    });
    named || holds_local_state || converging
}

impl PodHandle {
    /// Re-derive and apply the desired state for one partition from fresh
    /// point reads. Every watch event is just a signal to look again —
    /// convergence acts on observed durable state, never on remembered
    /// event payloads, so missed, reordered, or replayed events cannot
    /// corrupt local state.
    async fn converge(&self, partition: u32) -> Result<bool> {
        let handoff = self.store.get_handoff(partition).await?;
        let assignment = self.store.get_assignment(partition).await?;
        self.apply(partition, assignment.as_ref(), handoff.as_ref())
            .await
    }

    /// Bound on concurrent cache warms. Only the two warm sites in
    /// `apply` queue here; everything else a convergence does (fence,
    /// drain, release, ack) must stay prompt regardless of warm pressure.
    async fn acquire_warm_slot(&self) -> Result<SemaphorePermit<'_>> {
        self.warm_slots
            .acquire()
            .await
            .map_err(|_| Error::invalid_state("warm semaphore closed".to_string()))
    }

    /// Drive local state (cache warmth, write fence, acks, held set) to
    /// the desired state. Every transition is idempotent; the watch loop
    /// runs at most one convergence per partition at a time, so no two
    /// applications for the same partition ever interleave — applications
    /// for different partitions may run concurrently.
    ///
    /// Returns whether the application changed local state (a warm, a
    /// fence, a resume, a release). A convergence that merely confirms
    /// the partition is already settled is a read, and the run budget's
    /// progress signal must not count it — a pod whose coordination is
    /// wedged still no-op-converges its settled partitions successfully.
    async fn apply(
        &self,
        partition: u32,
        assignment: Option<&PartitionAssignment>,
        handoff: Option<&HandoffState>,
    ) -> Result<bool> {
        let pod = &self.config.pod_name;
        let desired = desired_state(pod, assignment, handoff);
        let mut did_work = false;

        // The pending-ownership window: this pod will be told to warm
        // once the drain completes, and everything the warm needs that
        // touches no shared state can get ready now. Deliberately not a
        // DesiredState — the derivation stays a pure ownership answer —
        // and deliberately not `did_work`: preparation is a hint, and
        // counting it as progress would let a pod that only ever
        // prepares look healthy to the budgets.
        if prepare_hinted(pod, handoff) {
            self.handler.prepare_acquire(partition).await;
        }

        match desired {
            DesiredState::Serving => {
                if !self
                    .warmed_partitions
                    .lock()
                    .expect("warmed partitions lock poisoned")
                    .contains_key(&partition)
                {
                    tracing::info!(pod, partition, "converging to Serving: warming");
                    let _warm_slot = self.acquire_warm_slot().await?;
                    if self.capture_changelog(partition, 0).await?.is_none() {
                        return Ok(did_work);
                    }
                    let start = Instant::now();
                    self.handler.warm_partition(partition).await?;
                    histogram!("personhog_coordination_partition_warm_ms", "trigger" => "restart")
                        .record(start.elapsed().as_secs_f64() * 1000.0);
                    self.warmed_partitions
                        .lock()
                        .expect("warmed partitions lock poisoned")
                        .insert(partition, WarmProvenance::Serving);
                    did_work = true;
                }
                // Whatever the branch above did, the pod is meant to be
                // serving this partition now — so let the handler repair
                // anything it needs and no longer has. Repair applied is
                // progress like any other applied work.
                if self.handler.verify_serving(partition).await? {
                    did_work = true;
                }
                // Resume any local fence regardless of which branch ran:
                // a crash-restart inside the TTL can leave a partition
                // fenced but unwarmed (re-fenced through the Drained arm
                // before the handoff was cancelled), and relying on the
                // handler's warm to lift the fence would make write
                // admission depend on an undocumented handler side
                // effect. Resuming after a warm that already unfenced is
                // an idempotent no-op.
                //
                // Clear the local record only once the handler has
                // actually resumed: `resume_partition` can fail (it may
                // re-take broker-side state), and forgetting the fence
                // first would leave the data plane fenced with no branch
                // left to re-enter — writes rejected forever while the
                // convergence reports success.
                if self
                    .fenced_partitions
                    .lock()
                    .expect("fenced partitions lock poisoned")
                    .contains(&partition)
                {
                    tracing::info!(pod, partition, "converging to Serving: resuming writes");
                    self.handler.resume_partition(partition).await?;
                    self.fenced_partitions
                        .lock()
                        .expect("fenced partitions lock poisoned")
                        .remove(&partition);
                    did_work = true;
                }
            }
            DesiredState::Drained { ack } => {
                // The coordinator only advances Freezing → Draining once
                // every router has FreezeAcked, so no new request can flow
                // from any router to this pod and the inflight==0 the drain
                // waits for is meaningful. The produce path awaits Kafka
                // delivery before returning, so "no inflight handlers"
                // implies "every acked write is durable in Kafka."
                let newly_fencing = !self
                    .fenced_partitions
                    .lock()
                    .expect("fenced partitions lock poisoned")
                    .contains(&partition);
                if newly_fencing {
                    tracing::info!(pod, partition, "converging to Drained: fencing + draining");
                    did_work = true;
                }
                // Record the fence before the call that applies it. The
                // handler fences as its first action and only then waits,
                // so a failure after that point would otherwise leave the
                // data plane fenced with nothing here to say so — and a
                // later convergence to Serving, seeing no fence recorded,
                // would skip the resume that lifts it. Recording early
                // only risks a redundant resume, which is a no-op.
                self.fenced_partitions
                    .lock()
                    .expect("fenced partitions lock poisoned")
                    .insert(partition);
                let start = Instant::now();
                self.handler.drain_partition_inflight(partition).await?;
                if newly_fencing {
                    // Only the first convergence does a real drain wait;
                    // later re-convergences are no-ops that would bury the
                    // signal in near-zero samples.
                    histogram!("personhog_coordination_partition_drain_ms")
                        .record(start.elapsed().as_secs_f64() * 1000.0);
                }
                if ack {
                    let handoff = handoff.expect("Drained state only derives from a handoff");
                    self.store
                        .put_drained_ack(&PodDrainedAck {
                            pod_name: pod.clone(),
                            partition,
                            acked_at: util::now_seconds(),
                            acked_at_ms: 0,
                            handoff_id: handoff.handoff_id.clone(),
                        })
                        .await?;
                    tracing::info!(pod, partition, "drained ack written");
                }
            }
            DesiredState::Acquiring => {
                let handoff = handoff.expect("Acquiring state only derives from a handoff");
                // Only a warm installed for *this* handoff satisfies its
                // warming. One left over from an earlier era — the pod
                // was this partition's owner or warming target before,
                // ownership moved away, and the intervening Released
                // convergence never ran — can hold values that predate
                // writes the changelog accepted since. A fresh warm
                // replays only from the writer's committed offset, so a
                // stale cache below that offset would keep hitting;
                // release first and rebuild from clean.
                let valid = self.warmed_partitions.lock().expect("warmed partitions lock poisoned").get(&partition).is_some_and(
                    |provenance| {
                        matches!(provenance, WarmProvenance::Handoff(id) if *id == handoff.handoff_id)
                    },
                );
                if !valid {
                    if self
                        .warmed_partitions
                        .lock()
                        .expect("warmed partitions lock poisoned")
                        .contains_key(&partition)
                    {
                        tracing::info!(
                            pod,
                            partition,
                            "converging to Acquiring: releasing a warm from an earlier era"
                        );
                        self.release_warm(partition).await?;
                    }
                    tracing::info!(pod, partition, "converging to Acquiring: warming");
                    let _warm_slot = self.acquire_warm_slot().await?;
                    if self
                        .capture_changelog(partition, handoff.config_epoch)
                        .await?
                        .is_none()
                    {
                        return Ok(did_work);
                    }
                    let start = Instant::now();
                    self.handler.warm_partition(partition).await?;
                    histogram!("personhog_coordination_partition_warm_ms", "trigger" => "handoff")
                        .record(start.elapsed().as_secs_f64() * 1000.0);
                    self.warmed_partitions
                        .lock()
                        .expect("warmed partitions lock poisoned")
                        .insert(
                            partition,
                            WarmProvenance::Handoff(handoff.handoff_id.clone()),
                        );
                    did_work = true;
                }
                // The warm above re-admits writes for this partition as
                // part of taking ownership, so clearing the record here
                // matches the data plane rather than diverging from it.
                self.fenced_partitions
                    .lock()
                    .expect("fenced partitions lock poisoned")
                    .remove(&partition);
                self.store
                    .put_warmed_ack(&PodWarmedAck {
                        pod_name: pod.clone(),
                        partition,
                        acked_at: util::now_seconds(),
                        acked_at_ms: 0,
                        handoff_id: handoff.handoff_id.clone(),
                    })
                    .await?;
                tracing::info!(pod, partition, "warmed ack written");
            }
            DesiredState::Released => {
                // Forgotten only after the handler returns, matching the
                // discipline the other arms document. Removing first put
                // a suspension point between forgetting and releasing: a
                // lane dropped there — or a release that failed — left
                // the partition in neither map, so no convergence ever
                // dispatched for it again and its cache, floors, and
                // producer leaked for the life of the process. Release is
                // idempotent, so a retry that re-runs it costs nothing.
                let was_warmed = self
                    .warmed_partitions
                    .lock()
                    .expect("warmed partitions lock poisoned")
                    .contains_key(&partition);
                let was_fenced = self
                    .fenced_partitions
                    .lock()
                    .expect("fenced partitions lock poisoned")
                    .contains(&partition);
                if was_warmed || was_fenced {
                    tracing::info!(pod, partition, "converging to Released: releasing");
                    self.release_warm(partition).await?;
                    self.fenced_partitions
                        .lock()
                        .expect("fenced partitions lock poisoned")
                        .remove(&partition);
                    counter!("personhog_coordination_partition_releases_total").increment(1);
                    self.drain_notify.notify_one();
                    did_work = true;
                }
            }
        }

        gauge!("personhog_coordination_partitions_held").set(self.held_partition_count() as f64);

        Ok(did_work)
    }

    async fn watch_handoff_loop(
        &self,
        mut stream: WatchStream,
        cancel: CancellationToken,
        initial: HashSet<u32>,
        progress: &AtomicBool,
    ) -> Result<()> {
        // The truth path: periodically re-derive the involved-partition
        // set from a fresh snapshot and re-dispatch each member,
        // repairing whatever the event path failed to deliver. First
        // pass one interval out — the caller has just seeded `initial`
        // from the same derivation.
        let mut reconcile_tick = tokio::time::interval_at(
            tokio::time::Instant::now() + self.config.reconcile_interval,
            self.config.reconcile_interval,
        );
        reconcile_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);

        // Convergence is single-flight per partition: at most one
        // `converge` future per partition lives in `lanes` at a time,
        // preserving the no-interleaving guarantee `apply` relies on,
        // while different partitions converge concurrently — a deploy
        // moving several partitions onto this pod warms them in
        // parallel instead of queueing each behind the last. A signal
        // arriving for a partition already converging parks in
        // `pending`; `converge` reads fresh state, so one re-run
        // absorbs any number of coalesced signals. Dropping this loop
        // future drops every in-flight convergence with it, so cancel
        // semantics are unchanged from converging inline.
        let mut lanes: FuturesUnordered<BoxFuture<'_, (u32, Trigger, Result<bool>)>> =
            FuturesUnordered::new();
        let mut in_flight: HashSet<u32> = HashSet::new();
        let mut pending: HashMap<u32, Trigger> = HashMap::new();

        // One nudge-driven pass per reconcile interval at most. A
        // producer that condemns again right after every heal would
        // otherwise drive passes at broker speed — and each successful
        // heal counts as applied work, resetting the budgets that exist
        // to catch exactly that wedge. A suppressed nudge falls back to
        // the tick, so a flap degrades to tick-rate healing, the
        // pre-nudge shape.
        let mut last_repair_pass: Option<tokio::time::Instant> = None;

        /// Resolves when the repair nudge fires, or never when none is
        /// wired, leaving the other arms in charge.
        async fn nudged(nudge: &Option<Arc<Notify>>) {
            match nudge {
                Some(nudge) => nudge.notified().await,
                None => std::future::pending().await,
            }
        }

        fn dispatch<'s>(
            handle: &'s PodHandle,
            partition: u32,
            trigger: Trigger,
            in_flight: &mut HashSet<u32>,
            pending: &mut HashMap<u32, Trigger>,
            lanes: &mut FuturesUnordered<BoxFuture<'s, (u32, Trigger, Result<bool>)>>,
        ) {
            if in_flight.contains(&partition) {
                // Coalesce, keeping the stricter error disposition: a
                // fatal Event request must not be downgraded by a later
                // budgeted Reconcile one.
                let entry = pending.entry(partition).or_insert(trigger);
                if trigger == Trigger::Event {
                    *entry = Trigger::Event;
                }
                return;
            }
            in_flight.insert(partition);
            lanes.push(Box::pin(async move {
                (partition, trigger, handle.converge(partition).await)
            }));
        }

        for partition in initial {
            // Seed convergence carries Event severity: a pod that cannot
            // establish its local state at loop start must not serve.
            dispatch(
                self,
                partition,
                Trigger::Event,
                &mut in_flight,
                &mut pending,
                &mut lanes,
            );
        }

        // Reconcile health is judged per tick window, keeping the budget
        // at the pass granularity it was sized for: a window where any
        // reconcile-triggered convergence failed — or whose own
        // durable-state snapshot failed — counts exactly one failure, so
        // a single partition persistently failing (a bad warm, say)
        // still exhausts the budget even while its neighbors converge
        // fine, while one blip failing several things at once cannot
        // burn the budget faster than real time. A window of only
        // successes resets the count; a window with no completed
        // reconcile convergence at all (one still in flight across the
        // tick) is neutral.
        let mut consecutive_reconcile_failures: u32 = 0;
        let mut window_err: Option<Error> = None;
        let mut window_ok = false;

        loop {
            tokio::select! {
                _ = cancel.cancelled() => return Ok(()),
                _ = reconcile_tick.tick() => {
                    // One budget count per tick at most, no matter how
                    // many things failed inside its window: the budget is
                    // sized in passes, and a single blip failing both a
                    // convergence and the following snapshot is one
                    // failed pass, not two.
                    let mut tick_err: Option<Error> = window_err.take();
                    if tick_err.is_none() && window_ok {
                        consecutive_reconcile_failures = 0;
                    }
                    window_ok = false;

                    // List failures are tolerated for the same reason as
                    // the router's reconcile arm: a failed pass leaves
                    // the pod as stale as one tick ago and the next
                    // success fully compensates, so a brief etcd blip
                    // must not take the data plane down with it. The
                    // budget bounds the reads-failing-while-lease-healthy
                    // mode.
                    match self.involved_partitions().await {
                        Ok((partitions, _)) => {
                            for partition in partitions {
                                dispatch(
                                    self,
                                    partition,
                                    Trigger::Reconcile,
                                    &mut in_flight,
                                    &mut pending,
                                    &mut lanes,
                                );
                            }
                        }
                        Err(e) => {
                            if tick_err.is_none() {
                                tick_err = Some(e);
                            } else {
                                tracing::warn!(
                                    pod = %self.config.pod_name,
                                    error = %e,
                                    "durable-state snapshot failed in an already-failed window"
                                );
                            }
                        }
                    }

                    if let Some(e) = tick_err {
                        consecutive_reconcile_failures += 1;
                        counter!(
                            "personhog_coordination_reconcile_failures_total",
                            "component" => "pod"
                        )
                        .increment(1);
                        tracing::warn!(
                            pod = %self.config.pod_name,
                            error = %e,
                            consecutive = consecutive_reconcile_failures,
                            budget = self.config.reconcile_failure_budget,
                            "pod reconcile failed; continuing on last-known state"
                        );
                        if consecutive_reconcile_failures >= self.config.reconcile_failure_budget {
                            return Err(e);
                        }
                    }
                }
                _ = nudged(&self.repair_nudge) => {
                    let now = tokio::time::Instant::now();
                    let cooling = last_repair_pass.is_some_and(|last| {
                        now.duration_since(last) < self.config.reconcile_interval
                    });
                    if cooling {
                        counter!("personhog_coordination_repair_passes_total", "outcome" => "suppressed")
                            .increment(1);
                        tracing::warn!(
                            pod = %self.config.pod_name,
                            "repair nudged again inside the cooldown; leaving it to the reconcile tick"
                        );
                    } else {
                        last_repair_pass = Some(now);
                        counter!("personhog_coordination_repair_passes_total", "outcome" => "run")
                            .increment(1);
                        tracing::info!(
                            pod = %self.config.pod_name,
                            "data-plane repair nudge; converging involved partitions"
                        );
                        // The same derivation the tick runs: the nudge
                        // carries no payload, and `verify_serving`
                        // repairs exactly the partitions that need it
                        // while the rest converge as no-ops. Reconcile
                        // severity throughout; a failed snapshot leaves
                        // repair to the tick, whose budget owns
                        // sustained failure.
                        match self.involved_partitions().await {
                            Ok((partitions, _)) => {
                                for partition in partitions {
                                    dispatch(
                                        self,
                                        partition,
                                        Trigger::Reconcile,
                                        &mut in_flight,
                                        &mut pending,
                                        &mut lanes,
                                    );
                                }
                            }
                            Err(e) => {
                                tracing::warn!(
                                    pod = %self.config.pod_name,
                                    error = %e,
                                    "repair-pass snapshot failed; leaving repair to the reconcile tick"
                                );
                            }
                        }
                    }
                }
                msg = stream.message() => {
                    let resp = util::live_watch_response(msg?, "handoff")?;
                    for event in resp.events() {
                        let mut unreadable = false;
                        // Convergence still reads durable state rather
                        // than trusting the payload; the payload only
                        // decides whether to look. See
                        // `event_concerns_pod` for what makes an event
                        // this pod's business.
                        let partition = match event.event_type() {
                            EventType::Put => match parse_watch_value::<HandoffState>(event) {
                                Ok(handoff) => {
                                    util::record_phase_watch_delivery(
                                        "pod",
                                        handoff.phase,
                                        handoff.phase_entered_at_ms,
                                    );
                                    let partition = handoff.partition;
                                    if event_concerns_pod(
                                        &self.config.pod_name,
                                        Some(&handoff),
                                        self.holds_local_state(partition),
                                        in_flight.contains(&partition),
                                    ) {
                                        Some(partition)
                                    } else {
                                        None
                                    }
                                }
                                Err(e) => {
                                    tracing::error!(pod = %self.config.pod_name, error = %e, "failed to parse handoff");
                                    unreadable = true;
                                    None
                                }
                            },
                            EventType::Delete => match event
                                .kv()
                                .and_then(|kv| from_utf8(kv.key()).ok())
                                .and_then(store::extract_partition_from_key)
                            {
                                Some(p)
                                    if event_concerns_pod(
                                        &self.config.pod_name,
                                        None,
                                        self.holds_local_state(p),
                                        in_flight.contains(&p),
                                    ) =>
                                {
                                    Some(p)
                                }
                                Some(_) => None,
                                None => {
                                    unreadable = true;
                                    None
                                }
                            },
                        };
                        util::record_handoff_event_disposition(match (partition, unreadable) {
                            (Some(_), _) => "converged",
                            (None, true) => "unreadable",
                            (None, false) => "skipped",
                        });
                        if let Some(partition) = partition {
                            dispatch(
                                self,
                                partition,
                                Trigger::Event,
                                &mut in_flight,
                                &mut pending,
                                &mut lanes,
                            );
                        }
                    }
                }
                Some((partition, trigger, result)) = lanes.next(), if !lanes.is_empty() => {
                    in_flight.remove(&partition);
                    match result {
                        Ok(did_work) => {
                            // Progress means *applied* work — a handler
                            // invoked, local serving state changed — never
                            // a successful read. A no-op convergence of an
                            // already-settled partition succeeds even on a
                            // pod whose coordination is otherwise wedged,
                            // and counting it would make the run budget
                            // unreachable in exactly that wedge.
                            if did_work {
                                progress.store(true, Ordering::SeqCst);
                            }
                            if trigger == Trigger::Reconcile {
                                window_ok = true;
                            }
                        }
                        Err(e) => match trigger {
                            // The pod was told about a specific durable
                            // transition and cannot honor it; serving on
                            // regardless would hold the handoff hostage
                            // with no signal to the coordinator.
                            Trigger::Event => return Err(e),
                            Trigger::Reconcile => {
                                tracing::warn!(
                                    pod = %self.config.pod_name,
                                    partition,
                                    error = %e,
                                    "reconcile-triggered convergence failed"
                                );
                                window_err = Some(e);
                            }
                        },
                    }
                    if let Some(next) = pending.remove(&partition) {
                        dispatch(self, partition, next, &mut in_flight, &mut pending, &mut lanes);
                    }
                }
            }
        }
    }
}

/// Why a convergence was dispatched — decides what its failure means.
/// Event-driven convergence (including the seed at loop start) failing is
/// fatal to the run: the pod cannot honor a specific durable transition
/// it was told about. Reconcile-tick convergence failing is tolerated
/// under the reconcile failure budget: the pod is merely as stale as one
/// tick, and the next pass compensates.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Trigger {
    Event,
    Reconcile,
}

/// Surrender the moment this pod's own registration disappears.
///
/// A lease revoked or expired out from under a pod deletes its keys
/// immediately, but the pod only learns on its next keepalive round.
/// The coordinator sees the deletion at once and can reassign inside
/// that gap, so the pod can be answering reads from a cache the new
/// owner is already changing. This closes the gap to a watch delivery.
///
/// It is an accelerator, not the guarantee: a stream that never
/// establishes, or dies, simply leaves detection to the keepalive margin
/// as before, which is why nothing here retries or escalates.
async fn watch_own_registration(
    store: Arc<PersonhogStore>,
    pod_name: String,
    authority: Arc<AuthorityClock>,
    end_session: CancellationToken,
    cancel: CancellationToken,
) {
    // Establishment is raced against the token just like the stream
    // reads below: the session teardown joins this task, so an etcd call
    // stalling here would otherwise hold up the join — and behind it the
    // self-fence — for as long as the stall lasts.
    let revision = tokio::select! {
        _ = cancel.cancelled() => return,
        revision = store.current_revision() => match revision {
            Ok(revision) => revision,
            Err(e) => {
                tracing::warn!(pod = %pod_name, error = %e, "registration watch unavailable");
                return;
            }
        },
    };
    let mut stream = tokio::select! {
        _ = cancel.cancelled() => return,
        stream = store.watch_pods_from(revision + 1) => match stream {
            Ok(stream) => stream,
            Err(e) => {
                tracing::warn!(pod = %pod_name, error = %e, "registration watch unavailable");
                return;
            }
        },
    };
    let registration_key = store.pod_registration_key(&pod_name);
    loop {
        let message = tokio::select! {
            _ = cancel.cancelled() => return,
            message = stream.message() => message,
        };
        let Ok(Some(response)) = message else { return };
        // A cancelled watcher delivers nothing further. This watch is
        // defense-in-depth: the session teardown it accelerates still
        // happens without it, so ending is safe.
        if response.canceled() {
            tracing::warn!(pod = %pod_name, "registration watch cancelled by etcd; watch ends");
            return;
        }
        for event in response.events() {
            if event.event_type() != EventType::Delete {
                continue;
            }
            // Exactly the key `register` writes — this is a prefix
            // watch, and matching anything looser (say, a final path
            // segment) would let an unrelated deletion under the prefix
            // cost this pod its session.
            let deleted_us = event
                .kv()
                .and_then(|kv| from_utf8(kv.key()).ok())
                .is_some_and(|key| key == registration_key);
            if deleted_us {
                counter!("personhog_coordination_registration_deleted_total").increment(1);
                tracing::error!(
                    pod = %pod_name,
                    "registration deleted; surrendering serving authority immediately"
                );
                // Deliberately redundant with the session teardown's
                // surrender: this one stops strong reads at watch
                // delivery rather than when teardown finishes.
                authority.surrender();
                // Surrendering alone would leave a pod that holds a live
                // lease, refuses every read, and never registers again —
                // silently idle with nothing to escalate. Ending the
                // session is what puts it back to work.
                end_session.cancel();
                return;
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::types::AssignmentStatus;

    const POD: &str = "pod-self";
    const OTHER: &str = "pod-other";
    const THIRD: &str = "pod-third";

    fn assignment(owner: &str) -> PartitionAssignment {
        PartitionAssignment {
            partition: 1,
            owner: owner.to_string(),
            advertise_address: None,
            status: AssignmentStatus::Active,
        }
    }

    /// An event a pod is party to must reach a convergence, by any of
    /// the three routes — and one it is not party to must not. The skip
    /// direction is the one worth having: every integration test passed
    /// before the scoping existed, so nothing else asserts it.
    #[test]
    fn an_event_reaches_only_the_pods_it_concerns() {
        let cases = [
            (
                "named as the old owner",
                Some(handoff(Some(POD), OTHER, HandoffPhase::Freezing)),
                false,
                false,
                true,
            ),
            (
                "named as the new owner",
                Some(handoff(Some(OTHER), POD, HandoffPhase::Warming)),
                false,
                false,
                true,
            ),
            (
                // A replacement overwrites the handoff key in place, so
                // an old owner dropped from the successor sees a put
                // naming two other pods. Only the fence it still holds
                // says the fence should come off.
                "unnamed but still holding a fence or a cache",
                Some(handoff(Some(OTHER), THIRD, HandoffPhase::Freezing)),
                true,
                false,
                true,
            ),
            (
                "unnamed but converging it right now",
                Some(handoff(Some(OTHER), THIRD, HandoffPhase::Freezing)),
                false,
                true,
                true,
            ),
            (
                "a deletion, while holding local state for it",
                None,
                true,
                false,
                true,
            ),
            (
                "a deletion, while still warming for it",
                None,
                false,
                true,
                true,
            ),
            (
                "another pods' handoff, holding nothing and converging nothing",
                Some(handoff(Some(OTHER), THIRD, HandoffPhase::Freezing)),
                false,
                false,
                false,
            ),
            (
                "a deletion for a partition this pod has nothing to do with",
                None,
                false,
                false,
                false,
            ),
        ];

        for (case, handoff, holds_local_state, converging, expected) in cases {
            assert_eq!(
                event_concerns_pod(POD, handoff.as_ref(), holds_local_state, converging),
                expected,
                "{case}"
            );
        }
    }

    fn handoff(old_owner: Option<&str>, new_owner: &str, phase: HandoffPhase) -> HandoffState {
        HandoffState {
            partition: 1,
            old_owner: old_owner.map(str::to_string),
            new_owner: new_owner.to_string(),
            phase,
            started_at: 0,
            handoff_id: "h-test".to_string(),
            freeze_quorum: None,
            freeze_quorum_ref: None,
            created_at_ms: 0,
            phase_entered_at_ms: 0,
            held_by: None,
            config_epoch: 0,
            new_owner_address: None,
        }
    }

    fn held(phase: HandoffPhase) -> HandoffState {
        let mut h = handoff(Some(POD), POD, phase);
        h.held_by = Some("m-test".to_string());
        h
    }

    #[test]
    fn only_a_pending_incoming_owner_is_hinted() {
        use HandoffPhase::*;
        let cases = [
            (
                "incoming, freezing",
                Some(handoff(Some("other"), POD, Freezing)),
                true,
            ),
            (
                "incoming, draining",
                Some(handoff(Some("other"), POD, Draining)),
                true,
            ),
            (
                "incoming, warming",
                Some(handoff(Some("other"), POD, Warming)),
                false,
            ),
            (
                "outgoing, draining",
                Some(handoff(Some(POD), "other", Draining)),
                false,
            ),
            ("held, draining", Some(held(Draining)), false),
            ("no handoff", None, false),
        ];
        for (name, h, expected) in cases {
            assert_eq!(prepare_hinted(POD, h.as_ref()), expected, "{name}");
        }
    }

    #[test]
    fn desired_state_covers_the_full_role_phase_matrix() {
        use DesiredState::*;
        use HandoffPhase::*;

        // (case, assignment, handoff, expected). Assignments mirror what
        // the protocol would hold at that point: they name the old owner
        // (or nobody) until Complete flips them to the new owner.
        let cases: Vec<(
            &str,
            Option<PartitionAssignment>,
            Option<HandoffState>,
            DesiredState,
        )> = vec![
            (
                "old owner serves through Freezing",
                Some(assignment(POD)),
                Some(handoff(Some(POD), OTHER, Freezing)),
                Serving,
            ),
            (
                "old owner drains and acks in Draining",
                Some(assignment(POD)),
                Some(handoff(Some(POD), OTHER, Draining)),
                Drained { ack: true },
            ),
            (
                "old owner stays drained without re-acking in Warming",
                Some(assignment(POD)),
                Some(handoff(Some(POD), OTHER, Warming)),
                Drained { ack: false },
            ),
            (
                "old owner releases at Complete",
                Some(assignment(OTHER)),
                Some(handoff(Some(POD), OTHER, Complete)),
                Released,
            ),
            (
                // A cancelled handoff is replaced by a reaffirm toward
                // the current owner, and the coordinator deliberately
                // leaves `old_owner` unset on it: naming this pod on
                // both sides would match the old-owner arm first and
                // release the partition instead of resuming it. That is
                // a silent partition drop, so the shape is pinned here
                // rather than left to a comment.
                "reaffirmed owner resumes rather than releasing",
                Some(assignment(POD)),
                Some(handoff(None, POD, Complete)),
                Serving,
            ),
            (
                "new owner must not hold the partition in Freezing",
                Some(assignment(OTHER)),
                Some(handoff(Some(OTHER), POD, Freezing)),
                Released,
            ),
            (
                "new owner must not warm early in Draining",
                Some(assignment(OTHER)),
                Some(handoff(Some(OTHER), POD, Draining)),
                Released,
            ),
            (
                "new owner acquires in Warming",
                Some(assignment(OTHER)),
                Some(handoff(Some(OTHER), POD, Warming)),
                Acquiring,
            ),
            (
                "new owner serves at Complete",
                Some(assignment(POD)),
                Some(handoff(Some(OTHER), POD, Complete)),
                Serving,
            ),
            (
                "initial assignment (no old owner) acquires in Warming",
                None,
                Some(handoff(None, POD, Warming)),
                Acquiring,
            ),
            (
                "held holder keeps serving through Freezing",
                Some(assignment(POD)),
                Some(held(Freezing)),
                Serving,
            ),
            (
                "held holder drains and acks in Draining",
                Some(assignment(POD)),
                Some(held(Draining)),
                Drained { ack: true },
            ),
            (
                "same-owner replacement re-acquires in Warming",
                Some(assignment(POD)),
                Some(handoff(Some(POD), POD, Warming)),
                Acquiring,
            ),
            (
                "same-owner replacement serves at Complete",
                Some(assignment(POD)),
                Some(handoff(Some(POD), POD, Complete)),
                Serving,
            ),
            (
                "handoff between two other pods defers to owned assignment",
                Some(assignment(POD)),
                Some(handoff(Some(OTHER), THIRD, Draining)),
                Serving,
            ),
            (
                "handoff between two other pods defers to unowned assignment",
                Some(assignment(OTHER)),
                Some(handoff(Some(OTHER), THIRD, Warming)),
                Released,
            ),
            (
                "no handoff: owned assignment serves",
                Some(assignment(POD)),
                None,
                Serving,
            ),
            (
                "no handoff: someone else's assignment releases",
                Some(assignment(OTHER)),
                None,
                Released,
            ),
            ("no assignment at all releases", None, None, Released),
        ];

        for (case, assignment, handoff, expected) in cases {
            assert_eq!(
                desired_state(POD, assignment.as_ref(), handoff.as_ref()),
                expected,
                "{case}"
            );
        }
    }
}