Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

131 KB
Raw Blame
mod common;

use std::sync::Arc;

use tokio_util::sync::CancellationToken;

use std::time::Duration;

use common::{
    start_coordinator, start_coordinator_named, start_coordinator_with_debounce, start_pod,
    start_pod_blocking, start_pod_slow, start_pod_with_lease_ttl, start_router,
    start_router_with_lease_ttl, test_store, wait_for_condition, HandoffEvent, PodHandles,
    POLL_INTERVAL, WAIT_TIMEOUT,
};
use personhog_coordination::strategy::{
    AssignmentStrategy, JumpHashStrategy, StickyBalancedStrategy,
};

const NUM_PARTITIONS: u32 = 8;
const MANY_PARTITIONS: u32 = 12;

/// Generates `#[tokio::test]` functions for each strategy variant.
/// Each test runs inside a `sticky_balanced` or `jump_hash` module.
///
/// Test functions receive `(test_name, strategy, evenly_distributed)` where
/// `evenly_distributed` indicates whether the strategy guarantees perfectly
/// balanced partition counts across pods.
macro_rules! strategy_tests {
    ($( $(#[$meta:meta])* $name:ident ),* $(,)?) => {
        mod sticky_balanced {
            use super::*;
            $(
                $(#[$meta])*
                #[tokio::test]
                async fn $name() {
                    super::$name(
                        concat!(stringify!($name), "-sticky"),
                        Arc::new(StickyBalancedStrategy),
                        true,
                    ).await;
                }
            )*
        }

        mod jump_hash {
            use super::*;
            $(
                $(#[$meta])*
                #[tokio::test]
                async fn $name() {
                    super::$name(
                        concat!(stringify!($name), "-jump"),
                        Arc::new(JumpHashStrategy),
                        false,
                    ).await;
                }
            )*
        }
    };
}

// Add tests here in order to test them for each strategy variant.
strategy_tests! {
    single_pod_gets_all_partitions,
    scale_up_triggers_handoff,
    pod_crash_reassigns_partitions,
    leader_failover,
    multi_router_ack_quorum,
    pod_crash_during_warming_restores_assignments,
    scale_down_to_one_pod,
    rapid_pod_joins,
    rolling_update,
    coordinator_starts_after_pods,
    debounce_batches_rapid_pod_changes,
    graceful_drain_transfers_partitions,
}

async fn single_pod_gets_all_partitions(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for the full bootstrap to settle: assignments written AND
    // handoffs cleaned up. Under the unified protocol, initial assignments
    // also go through a handoff cycle that finishes by deleting the
    // handoff record.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-0")
                && handoffs.is_empty()
        }
    })
    .await;

    let assignments = store.list_assignments().await.unwrap();
    assert_eq!(assignments.len(), NUM_PARTITIONS as usize);
    for a in &assignments {
        assert_eq!(a.owner, "writer-0");
    }

    let handoffs = store.list_handoffs().await.unwrap();
    assert!(handoffs.is_empty());

    cancel.cancel();
}

async fn scale_up_triggers_handoff(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Start coordinator, one pod, and one router
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Wait for initial assignment: all partitions to writer-0
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-0")
        }
    })
    .await;

    // Add a second pod — triggers rebalance
    let pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());

    // Wait for assignments to be split between both pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let has_pod0 = assignments.iter().any(|a| a.owner == "writer-0");
            let has_pod1 = assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize && has_pod0 && has_pod1
        }
    })
    .await;

    // Wait for all handoffs to complete (deleted from etcd)
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Verify: pod1 warmed the partitions it received
    let pod1_events = pod1.events.lock().await;
    let warmed: Vec<u32> = pod1_events
        .iter()
        .filter_map(|e| match e {
            HandoffEvent::Warmed(p) => Some(*p),
            _ => None,
        })
        .collect();
    assert!(
        !warmed.is_empty(),
        "pod1 should have warmed at least one partition"
    );

    // Verify: pod0 released the partitions that moved away
    let pod0_events = pod0.events.lock().await;
    let released: Vec<u32> = pod0_events
        .iter()
        .filter_map(|e| match e {
            HandoffEvent::Released(p) => Some(*p),
            _ => None,
        })
        .collect();
    assert!(
        !released.is_empty(),
        "pod0 should have released at least one partition"
    );

    // Verify: router executed cutovers
    let router_events = router.events.lock().await;
    assert!(
        !router_events.is_empty(),
        "router should have executed at least one cutover"
    );

    // Under the unified protocol, the router observes handoff events for
    // both the initial assignment (new_owner=writer-0) and the subsequent
    // reassignment (new_owner=writer-1). All targets must be a known live
    // pod; we just need to see at least one cutover involving writer-1 to
    // prove the scale-up reassignment actually happened.
    let saw_writer1_target = router_events.iter().any(|e| match e {
        common::CutoverEvent::StashBegan { new_owner, .. } => new_owner == "writer-1",
        common::CutoverEvent::StashDrained { target, .. } => target == "writer-1",
    });
    assert!(
        saw_writer1_target,
        "router should have observed at least one cutover targeting writer-1"
    );
    for event in router_events.iter() {
        match event {
            common::CutoverEvent::StashBegan { new_owner, .. } => {
                assert!(
                    new_owner == "writer-0" || new_owner == "writer-1",
                    "stash-began target should be a known pod, got {new_owner}"
                );
            }
            common::CutoverEvent::StashDrained { target, .. } => {
                assert!(
                    target == "writer-0" || target == "writer-1",
                    "stash-drained target should be a known pod, got {target}"
                );
            }
        }
    }

    // Router's internal routing table should match etcd assignments
    let final_assignments = store.list_assignments().await.unwrap();
    let expected: std::collections::HashMap<u32, String> = final_assignments
        .iter()
        .map(|a| (a.partition, a.owner.clone()))
        .collect();
    let router_table = router.table.read().await;
    assert_eq!(
        *router_table, expected,
        "router routing table should match etcd assignments"
    );

    cancel.cancel();
}

async fn pod_crash_reassigns_partitions(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Start coordinator and two pods (short lease so crash is detected fast)
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Use a short lease for pod0 so crash is detected quickly
    let pod0_cancel = CancellationToken::new();
    let _pod0 = start_pod_with_lease_ttl(Arc::clone(&store), "writer-0", 5, pod0_cancel.clone());
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());

    // Wait for balanced assignment across both pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let has_both = assignments.iter().any(|a| a.owner == "writer-0")
                && assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize && has_both
        }
    })
    .await;

    // Wait for handoffs from initial rebalance to settle
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Kill pod0 — its lease will expire and keys will be deleted
    pod0_cancel.cancel();

    // Wait for the pod registration to disappear (lease expiry)
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let pods = store.list_pods().await.unwrap_or_default();
            pods.len() == 1 && pods[0].pod_name == "writer-1"
        }
    })
    .await;

    // Wait for all partitions to be reassigned to the surviving pod
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-1")
        }
    })
    .await;

    cancel.cancel();
}

async fn leader_failover(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Start coordinator-0 with short lease so failover is fast
    let leader_cancel = CancellationToken::new();
    let _leader = start_coordinator_named(
        Arc::clone(&store),
        "coordinator-0",
        2,
        Arc::clone(&strategy),
        leader_cancel.clone(),
    );

    // Start a pod and router
    let _pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Wait for coordinator-0 to become leader and write assignments
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let is_leader = store
                .get_leader()
                .await
                .ok()
                .flatten()
                .is_some_and(|l| l.holder == "coordinator-0");
            let has_assignments =
                store.list_assignments().await.unwrap_or_default().len() == NUM_PARTITIONS as usize;
            is_leader && has_assignments
        }
    })
    .await;

    // Now start the standby — coordinator-0 already holds the leader key
    let _standby = start_coordinator_named(
        Arc::clone(&store),
        "coordinator-1",
        10,
        strategy,
        cancel.clone(),
    );

    // Kill the leader
    leader_cancel.cancel();

    // Wait for the standby to win the election
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            store
                .get_leader()
                .await
                .ok()
                .flatten()
                .is_some_and(|l| l.holder == "coordinator-1")
        }
    })
    .await;

    // Add a second pod — the new leader must handle this
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());

    // Wait for rebalance across both pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let has_both = assignments.iter().any(|a| a.owner == "writer-0")
                && assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize && has_both
        }
    })
    .await;

    cancel.cancel();
}

async fn multi_router_ack_quorum(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
    let router0 = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let router1 = start_router(Arc::clone(&store), "router-1", cancel.clone());

    // Wait for initial assignment and both routers to register.
    // Routers must have their handoff watch streams active before any
    // handoffs are created, otherwise they miss the Ready event.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assigned =
                store.list_assignments().await.unwrap_or_default().len() == NUM_PARTITIONS as usize;
            let routers = store.list_routers().await.unwrap_or_default().len() == 2;
            assigned && routers
        }
    })
    .await;

    // Add second pod to trigger handoffs
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());

    // Wait for assignments to be split between both pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let has_pod0 = assignments.iter().any(|a| a.owner == "writer-0");
            let has_pod1 = assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize && has_pod0 && has_pod1
        }
    })
    .await;

    // Wait for handoff cleanup to complete
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Both routers should have executed cutovers
    let r0_events = router0.events.lock().await;
    let r1_events = router1.events.lock().await;
    assert!(
        !r0_events.is_empty(),
        "router-0 should have executed cutovers"
    );
    assert!(
        !r1_events.is_empty(),
        "router-1 should have executed cutovers"
    );

    // Both routers should have cut over the same partitions
    fn event_partition(e: &common::CutoverEvent) -> u32 {
        match e {
            common::CutoverEvent::StashBegan { partition, .. }
            | common::CutoverEvent::StashDrained { partition, .. } => *partition,
        }
    }
    let r0_partitions: std::collections::HashSet<u32> =
        r0_events.iter().map(event_partition).collect();
    let r1_partitions: std::collections::HashSet<u32> =
        r1_events.iter().map(event_partition).collect();
    assert_eq!(r0_partitions, r1_partitions);

    // Both routers should have identical routing tables matching etcd assignments
    let final_assignments = store.list_assignments().await.unwrap();
    let expected: std::collections::HashMap<u32, String> = final_assignments
        .iter()
        .map(|a| (a.partition, a.owner.clone()))
        .collect();

    let r0_table = router0.table.read().await;
    let r1_table = router1.table.read().await;
    assert_eq!(
        *r0_table, expected,
        "router-0 routing table should match etcd assignments"
    );
    assert_eq!(
        *r1_table, expected,
        "router-1 routing table should match etcd assignments"
    );

    cancel.cancel();
}

async fn pod_crash_during_warming_restores_assignments(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Start 5 pods
    let mut pods: Vec<PodHandles> = Vec::new();
    for i in 0..5 {
        pods.push(start_pod(
            Arc::clone(&store),
            &format!("writer-{i}"),
            cancel.clone(),
        ));
    }

    // Wait for all partitions to be assigned across 5 pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            if assignments.len() != MANY_PARTITIONS as usize {
                return false;
            }
            // All 5 pods should have at least 1 partition
            (0..5).all(|i| {
                let name = format!("writer-{i}");
                assignments.iter().any(|a| a.owner == name)
            })
        }
    })
    .await;

    // Wait for any initial handoffs to settle
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Snapshot the stable assignments before adding the 6th pod
    let stable_assignments: std::collections::HashMap<u32, String> = store
        .list_assignments()
        .await
        .unwrap()
        .into_iter()
        .map(|a| (a.partition, a.owner))
        .collect();

    // Add a 6th pod with a blocking warm handler and short lease.
    // The blocking handler ensures the handoff stays at Warming when pod-5 dies.
    let pod5_cancel = CancellationToken::new();
    let _pod5 = start_pod_blocking(Arc::clone(&store), "writer-5", 2, pod5_cancel.clone());

    // Wait for writer-5 to register
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            store
                .list_pods()
                .await
                .unwrap_or_default()
                .iter()
                .any(|p| p.pod_name == "writer-5")
        }
    })
    .await;

    // Kill pod-5 immediately — it should still be in the Warming phase
    pod5_cancel.cancel();

    // Wait for writer-5's lease to expire
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            !store
                .list_pods()
                .await
                .unwrap_or_default()
                .iter()
                .any(|p| p.pod_name == "writer-5")
        }
    })
    .await;

    // Stale handoffs targeting the dead pod should be cleaned up
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Assignments should be back to the original 5-pod distribution
    let final_assignments: std::collections::HashMap<u32, String> = store
        .list_assignments()
        .await
        .unwrap()
        .into_iter()
        .map(|a| (a.partition, a.owner))
        .collect();

    assert_eq!(
        final_assignments, stable_assignments,
        "assignments should revert to pre-crash state"
    );

    // No partition should be owned by the dead pod
    assert!(
        !final_assignments.values().any(|v| v == "writer-5"),
        "dead pod should not own any partitions"
    );

    cancel.cancel();
}

async fn scale_down_to_one_pod(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Start 3 pods with short leases so crash detection is fast
    let pod0_cancel = CancellationToken::new();
    let _pod0 = start_pod_with_lease_ttl(Arc::clone(&store), "writer-0", 5, pod0_cancel.clone());
    let pod1_cancel = CancellationToken::new();
    let _pod1 = start_pod_with_lease_ttl(Arc::clone(&store), "writer-1", 5, pod1_cancel.clone());
    let _pod2 = start_pod(Arc::clone(&store), "writer-2", cancel.clone());

    // Wait for balanced assignment across all 3 pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            assignments.len() == MANY_PARTITIONS as usize
                && ["writer-0", "writer-1", "writer-2"]
                    .iter()
                    .all(|name| assignments.iter().any(|a| a.owner == *name))
        }
    })
    .await;

    // Wait for initial handoffs to settle
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Kill pod-0
    pod0_cancel.cancel();

    // Wait for pod-0 to disappear and its partitions to move
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let pods = store.list_pods().await.unwrap_or_default();
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            !pods.iter().any(|p| p.pod_name == "writer-0")
                && assignments.len() == MANY_PARTITIONS as usize
                && !assignments.iter().any(|a| a.owner == "writer-0")
                && handoffs.is_empty()
        }
    })
    .await;

    // Kill pod-1
    pod1_cancel.cancel();

    // Wait for all partitions to converge to the sole survivor
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let pods = store.list_pods().await.unwrap_or_default();
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            pods.len() == 1
                && assignments.len() == MANY_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-2")
                && handoffs.is_empty()
        }
    })
    .await;

    cancel.cancel();
}

async fn rapid_pod_joins(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(Arc::clone(&store), Arc::clone(&strategy), cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Start with one pod and wait for it to get all partitions + router registered
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let routers = store.list_routers().await.unwrap_or_default();
            assignments.len() == MANY_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-0")
                && routers.len() == 1
        }
    })
    .await;

    // Rapidly add 3 more pods without waiting between them
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());
    let _pod2 = start_pod(Arc::clone(&store), "writer-2", cancel.clone());
    let _pod3 = start_pod(Arc::clone(&store), "writer-3", cancel.clone());

    let pod_names: Vec<String> = (0..4).map(|i| format!("writer-{i}")).collect();

    // Wait until the strategy is satisfied: all partitions assigned, no handoffs,
    // every pod owns partitions, and the strategy would make no further changes.
    let check_store = Arc::clone(&store);
    let check_strategy = Arc::clone(&strategy);
    let check_pods = pod_names.clone();
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        let strategy = Arc::clone(&check_strategy);
        let pods = check_pods.clone();
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            if assignments.len() != MANY_PARTITIONS as usize || !handoffs.is_empty() {
                return false;
            }
            let current: std::collections::HashMap<u32, String> = assignments
                .iter()
                .map(|a| (a.partition, a.owner.clone()))
                .collect();
            let members = personhog_coordination::strategy::Member::active_all(&pods);
            let desired = strategy.compute_assignments(&current, &members, MANY_PARTITIONS);
            assignment_coordination::util::compute_required_handoffs(&current, &desired).is_empty()
        }
    })
    .await;

    // Verify every pod owns at least one partition
    let assignments = store.list_assignments().await.unwrap();
    for name in &pod_names {
        assert!(
            assignments.iter().any(|a| a.owner == *name),
            "{name} should own at least one partition"
        );
    }

    if evenly_distributed {
        let per_pod = MANY_PARTITIONS as usize / pod_names.len();
        for name in &pod_names {
            assert_eq!(
                assignments.iter().filter(|a| a.owner == *name).count(),
                per_pod,
                "expected even distribution for {name}"
            );
        }
    }

    cancel.cancel();
}

/// Simulates a Kubernetes rolling update: 3 old pods are replaced one at a time
/// by 3 new pods. Each time an old pod dies, a new pod spawns immediately.
///
/// Flow: start old-{0,1,2} -> for each i: new-i starts, old-i dies, new-(i+1) starts...
/// Final state: all partitions on new-{0,1,2}.
async fn rolling_update(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Start 3 "old generation" pods with short leases for fast crash detection
    let old_cancels: Vec<CancellationToken> = (0..3).map(|_| CancellationToken::new()).collect();
    let mut _old_pods: Vec<PodHandles> = Vec::new();
    for (i, old_cancel) in old_cancels.iter().enumerate() {
        _old_pods.push(start_pod_with_lease_ttl(
            Arc::clone(&store),
            &format!("old-{i}"),
            5,
            old_cancel.clone(),
        ));
    }

    // Wait for router to register and balanced assignment across all 3 old pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let routers = store.list_routers().await.unwrap_or_default();
            assignments.len() == MANY_PARTITIONS as usize
                && routers.len() == 1
                && (0..3).all(|i| {
                    let name = format!("old-{i}");
                    assignments.iter().any(|a| a.owner == name)
                })
        }
    })
    .await;

    // Wait for initial handoffs to settle
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Rolling replacement: for each old pod, start a new pod then kill the old one.
    // The next new pod spawns as soon as the old pod dies.
    let mut _new_pods: Vec<PodHandles> = Vec::new();
    for (i, old_cancel) in old_cancels.iter().enumerate() {
        // New pod comes up (temporarily 4 pods)
        _new_pods.push(start_pod(
            Arc::clone(&store),
            &format!("new-{i}"),
            cancel.clone(),
        ));

        // Wait for the new pod to register and handoffs from scale-up to settle
        let check_store = Arc::clone(&store);
        let new_name = format!("new-{i}");
        wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
            let store = Arc::clone(&check_store);
            let new_name = new_name.clone();
            async move {
                let pods = store.list_pods().await.unwrap_or_default();
                let handoffs = store.list_handoffs().await.unwrap_or_default();
                pods.iter().any(|p| p.pod_name == new_name) && handoffs.is_empty()
            }
        })
        .await;

        // Kill the old pod (back to 3 pods)
        old_cancel.cancel();

        // Wait for old pod lease to expire, reassignment, and handoffs to settle
        let check_store = Arc::clone(&store);
        let old_name = format!("old-{i}");
        wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
            let store = Arc::clone(&check_store);
            let old_name = old_name.clone();
            async move {
                let pods = store.list_pods().await.unwrap_or_default();
                let assignments = store.list_assignments().await.unwrap_or_default();
                let handoffs = store.list_handoffs().await.unwrap_or_default();
                !pods.iter().any(|p| p.pod_name == old_name)
                    && !assignments.iter().any(|a| a.owner == old_name)
                    && assignments.len() == MANY_PARTITIONS as usize
                    && handoffs.is_empty()
            }
        })
        .await;
    }

    // Final state: all partitions exclusively on new-{0,1,2}
    let assignments = store.list_assignments().await.unwrap();
    assert_eq!(assignments.len(), MANY_PARTITIONS as usize);

    let new_pod_names: std::collections::HashSet<&str> =
        ["new-0", "new-1", "new-2"].into_iter().collect();
    for a in &assignments {
        assert!(
            new_pod_names.contains(a.owner.as_str()),
            "partition {} owned by '{}', expected one of {:?}",
            a.partition,
            a.owner,
            new_pod_names,
        );
    }

    // No old pods should be registered
    let pods = store.list_pods().await.unwrap();
    assert_eq!(pods.len(), 3);
    for pod in &pods {
        assert!(
            pod.pod_name.starts_with("new-"),
            "expected only new pods, found '{}'",
            pod.pod_name,
        );
    }

    cancel.cancel();
}

async fn coordinator_starts_after_pods(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Start pods and router BEFORE the coordinator
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Wait for both pods and router to register
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let pods = store.list_pods().await.unwrap_or_default();
            let routers = store.list_routers().await.unwrap_or_default();
            pods.len() == 2 && routers.len() == 1
        }
    })
    .await;

    // No assignments yet — nobody is coordinating
    let assignments = store.list_assignments().await.unwrap();
    assert!(
        assignments.is_empty(),
        "no coordinator means no assignments"
    );

    // Now start the coordinator — it should discover existing pods
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    // Wait for assignments to be split between both pods
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let has_pod0 = assignments.iter().any(|a| a.owner == "writer-0");
            let has_pod1 = assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize
                && has_pod0
                && has_pod1
                && handoffs.is_empty()
        }
    })
    .await;

    cancel.cancel();
}

/// Tests that the debounce window batches rapid pod registrations into fewer
/// rebalances, and that the system converges correctly even when handoffs take
/// longer than the debounce interval.
///
/// Setup: 500ms debounce, 300ms warm delay per partition.
/// 1. Start pod-0, wait for all partitions assigned
/// 2. Rapidly add pod-1, pod-2, pod-3 (within debounce window)
/// 3. Verify convergence: all pods own partitions, no handoffs stuck
async fn debounce_batches_rapid_pod_changes(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator_with_debounce(
        Arc::clone(&store),
        Arc::clone(&strategy),
        Duration::from_millis(500),
        cancel.clone(),
    );
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Pod-0 uses a slow handler: 300ms per partition warm
    let warm_delay = Duration::from_millis(300);
    let _pod0 = start_pod_slow(Arc::clone(&store), "writer-0", warm_delay, cancel.clone());

    // Wait for initial assignment + router registered
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let routers = store.list_routers().await.unwrap_or_default();
            assignments.len() == MANY_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-0")
                && routers.len() == 1
        }
    })
    .await;

    // Rapidly add 3 more slow pods — all within the 500ms debounce window
    let _pod1 = start_pod_slow(Arc::clone(&store), "writer-1", warm_delay, cancel.clone());
    let _pod2 = start_pod_slow(Arc::clone(&store), "writer-2", warm_delay, cancel.clone());
    let _pod3 = start_pod_slow(Arc::clone(&store), "writer-3", warm_delay, cancel.clone());

    let pod_names: Vec<String> = (0..4).map(|i| format!("writer-{i}")).collect();

    // Wait for the strategy to be satisfied (no further handoffs needed).
    // Longer timeout: slow warm (300ms * partitions) + debounce (500ms) + handoff rounds
    let check_store = Arc::clone(&store);
    let check_strategy = Arc::clone(&strategy);
    let check_pods = pod_names.clone();
    wait_for_condition(Duration::from_secs(30), POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        let strategy = Arc::clone(&check_strategy);
        let pods = check_pods.clone();
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            if assignments.len() != MANY_PARTITIONS as usize || !handoffs.is_empty() {
                return false;
            }
            let current: std::collections::HashMap<u32, String> = assignments
                .iter()
                .map(|a| (a.partition, a.owner.clone()))
                .collect();
            let members = personhog_coordination::strategy::Member::active_all(&pods);
            let desired = strategy.compute_assignments(&current, &members, MANY_PARTITIONS);
            assignment_coordination::util::compute_required_handoffs(&current, &desired).is_empty()
        }
    })
    .await;

    // Verify every pod owns at least one partition
    let assignments = store.list_assignments().await.unwrap();
    for name in &pod_names {
        assert!(
            assignments.iter().any(|a| a.owner == *name),
            "{name} should own at least one partition"
        );
    }

    if evenly_distributed {
        let per_pod = MANY_PARTITIONS as usize / pod_names.len();
        for name in &pod_names {
            assert_eq!(
                assignments.iter().filter(|a| a.owner == *name).count(),
                per_pod,
                "expected even distribution for {name}"
            );
        }
    }

    cancel.cancel();
}

/// Verify that a pod going through graceful drain (SIGTERM) hands off its
/// partitions to surviving pods before exiting.
///
/// Flow:
/// 1. Two pods share partitions.
/// 2. Pod-0 receives cancel (simulating SIGTERM) → sets status to Draining.
/// 3. Coordinator sees Draining, creates handoffs to pod-1.
/// 4. Pod-0 releases partitions after handoff completes.
/// 5. Pod-1 ends up owning all partitions.
async fn graceful_drain_transfers_partitions(
    test_name: &str,
    strategy: Arc<dyn AssignmentStrategy>,
    _evenly_distributed: bool,
) {
    let store = test_store(test_name).await;
    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let coord_cancel = CancellationToken::new();
    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::clone(&strategy),
        coord_cancel.clone(),
    );
    let _router = start_router(Arc::clone(&store), "router-0", coord_cancel.clone());

    // Start two pods, each with its own cancel token
    let pod0_cancel = CancellationToken::new();
    let pod0 = start_pod(Arc::clone(&store), "writer-0", pod0_cancel.clone());
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", coord_cancel.clone());

    // Wait for both pods to have partitions assigned
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().any(|a| a.owner == "writer-0")
                && assignments.iter().any(|a| a.owner == "writer-1")
                && handoffs.is_empty()
        }
    })
    .await;

    // Cancel pod-0 (simulates SIGTERM) — it should drain gracefully
    pod0_cancel.cancel();

    // Wait for pod-0 to transition away from Ready
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let pods = store.list_pods().await.unwrap_or_default();
            !pods.iter().any(|p| {
                p.pod_name == "writer-0"
                    && p.status == personhog_coordination::types::PodStatus::Ready
            })
        }
    })
    .await;

    // Wait for all partitions to be owned by pod-1 (handoffs complete)
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-1")
                && handoffs.is_empty()
        }
    })
    .await;

    // Wait for pod-0 to record Released events (may lag behind handoff deletion)
    let check_events = Arc::clone(&pod0.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_events);
        async move {
            events
                .lock()
                .await
                .iter()
                .any(|e| matches!(e, HandoffEvent::Released(_)))
        }
    })
    .await;

    // Verify all partitions belong to pod-1
    let assignments = store.list_assignments().await.unwrap();
    assert_eq!(assignments.len(), NUM_PARTITIONS as usize);
    for a in &assignments {
        assert_eq!(
            a.owner, "writer-1",
            "partition {} should be owned by writer-1",
            a.partition
        );
    }

    coord_cancel.cancel();
}

/// Verify that when the drain status write to etcd fails (e.g. pod key was
/// already deleted), the pod exits cleanly without hanging or panicking.
/// It falls back to lease-based cleanup.
#[tokio::test]
async fn drain_status_write_failure_exits_cleanly() {
    let store = test_store("drain-write-fail").await;
    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let coord_cancel = CancellationToken::new();
    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::clone(&strategy),
        coord_cancel.clone(),
    );
    let _router = start_router(Arc::clone(&store), "router-0", coord_cancel.clone());

    let pod0_cancel = CancellationToken::new();
    let pod0 = start_pod(Arc::clone(&store), "writer-0", pod0_cancel.clone());

    // Wait for pod-0 to get all partitions
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-0")
        }
    })
    .await;

    // Delete the pod's key from etcd before cancelling. This simulates
    // the lease expiring or etcd state being lost. When drain() tries
    // update_pod_status(), it will fail with NotFound.
    store.delete_pod("writer-0").await.unwrap();

    // Cancel pod-0 (simulates SIGTERM)
    pod0_cancel.cancel();

    // The pod should exit cleanly within a reasonable time, not hang.
    let join_handle = pod0.join_handle.unwrap();
    let result = tokio::time::timeout(Duration::from_secs(5), join_handle)
        .await
        .expect("pod should exit within 5s, not hang")
        .expect("pod task should not panic");
    // drain() failure is logged as a warning, run() still returns Ok
    assert!(
        result.is_ok(),
        "pod should exit cleanly despite drain failure"
    );

    coord_cancel.cancel();
}

// ── New protocol tests (Patch 5: unified handoff) ───────────────────────

/// Initial assignments go through the full handoff protocol with
/// `old_owner = None`, not a direct-write path. Verifies the handoff
/// transitions through Freezing → Warming → Complete and that the pod
/// observes a Warmed event but never a Drained event (no old owner).
#[tokio::test]
async fn initial_assignment_creates_handoff_with_no_old_owner() {
    let store = test_store("initial-no-old-owner").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for full bootstrap to settle
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // The pod should have warmed every partition but never been asked to
    // drain (no partition had an old_owner).
    let events = pod.events.lock().await;
    let warmed_count = events
        .iter()
        .filter(|e| matches!(e, HandoffEvent::Warmed(_)))
        .count();
    let drained_count = events
        .iter()
        .filter(|e| matches!(e, HandoffEvent::Drained(_)))
        .count();
    let released_count = events
        .iter()
        .filter(|e| matches!(e, HandoffEvent::Released(_)))
        .count();

    assert_eq!(
        warmed_count, NUM_PARTITIONS as usize,
        "pod should have warmed every partition"
    );
    assert_eq!(
        drained_count, 0,
        "pod should not have been asked to drain (no old owner)"
    );
    assert_eq!(
        released_count, 0,
        "pod should not have released (no old owner)"
    );

    cancel.cancel();
}

/// During Freezing/Warming, the old owner retains its partition. It only
/// releases at Complete. This test starts a 2-pod cluster, adds a 3rd pod
/// to trigger reassignments, and asserts that Released events for the old
/// owner only fire after the corresponding Warmed events for the new owner.
#[tokio::test]
async fn old_owner_retains_partition_through_warming() {
    let store = test_store("retain-through-warming").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for writer-0 to own all partitions
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // Add writer-1 — triggers reassignments
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());

    // Wait for the reassignments to fully complete
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let has_both = assignments.iter().any(|a| a.owner == "writer-0")
                && assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize && has_both && handoffs.is_empty()
        }
    })
    .await;

    // For each partition that writer-0 released, the corresponding Drained
    // event must precede the Released event. (Released only fires at
    // Complete, while Drained fires at Freezing — earlier in the protocol.)
    let events = pod0.events.lock().await;
    for (i, event) in events.iter().enumerate() {
        if let HandoffEvent::Released(p) = event {
            let drained_index = events[..i]
                .iter()
                .position(|e| matches!(e, HandoffEvent::Drained(q) if q == p));
            assert!(
                drained_index.is_some(),
                "partition {p} was Released without a prior Drained event"
            );
        }
    }

    cancel.cancel();
}

/// A handoff toward a dead new owner is cancelled by the coordinator's
/// own replacement path: the planner reaffirms the partition to its live
/// current owner, and the router resolves the stash through its ordinary
/// Complete handling — draining home.
///
/// Verified by manually injecting and deleting a handoff via the store —
/// avoids the flakiness of relying on a blocking pod's lease to expire,
/// which is brittle because the pod's cancellation can't preempt a stuck
/// `warm_partition`.
#[tokio::test]
async fn a_dead_new_owner_handoff_is_reaffirmed_to_the_current_owner() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("delete-drains-stash").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for writer-0 to own all partitions
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // Inject a Freezing handoff for partition 0, targeting a fictitious
    // pod that will never write a WarmedAck. This simulates a "stuck"
    // handoff without needing a real blocking pod.
    let stuck_handoff = HandoffState {
        partition: 0,
        old_owner: Some("writer-0".to_string()),
        new_owner: "phantom-pod".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&stuck_handoff).await.unwrap();

    // Wait for the router to observe the handoff and call begin_stash
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            events
                .lock()
                .await
                .iter()
                .any(|e| matches!(e, common::CutoverEvent::StashBegan { partition: 0, .. }))
        }
    })
    .await;

    // Snapshot drain count *before* the Delete. The router has
    // already observed any drain_stash calls produced by the
    // bootstrap (each initial handoff completes via the Complete
    // path, which calls drain_stash with target=writer-0), so a
    // strict count comparison — not an event-existence check — is
    // required to attribute the next drain to the Delete event.
    let drains_before = router
        .events
        .lock()
        .await
        .iter()
        .filter(|e| matches!(e, common::CutoverEvent::StashDrained { .. }))
        .count();

    // No deletion is injected: the coordinator's own reconcile tick
    // detects the dead new owner, wakes the planner, and the planner
    // cancels the handoff by replacement — a reaffirm Complete toward
    // the live current owner, since the placement needs no move.

    // Wait for the drain *count* to grow past `drains_before`. An
    // event-existence check would race the bootstrap drains: those
    // already record `StashDrained{partition: 0, target: writer-0}`,
    // so any "does an event matching this exist" predicate would
    // return true immediately without waiting for the reaffirm-driven
    // drain to fire.
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            let count = events
                .lock()
                .await
                .iter()
                .filter(|e| matches!(e, common::CutoverEvent::StashDrained { .. }))
                .count();
            count > drains_before
        }
    })
    .await;

    // Sanity-check that the latest drain landed on the right target.
    // Because the count is now strictly larger than the bootstrap
    // baseline, at least one additional `StashDrained` must exist;
    // the Delete branch always targets the routing-table's current
    // owner, which is `writer-0` for partition 0.
    let events = router.events.lock().await;
    let post_delete_drain = events
        .iter()
        .rev()
        .find(|e| matches!(e, common::CutoverEvent::StashDrained { partition: 0, .. }))
        .expect("a Delete-triggered drain for partition 0 must exist");
    match post_delete_drain {
        common::CutoverEvent::StashDrained { target, .. } => {
            assert_eq!(target, "writer-0", "drain must target the current owner");
        }
        _ => unreachable!(),
    }

    cancel.cancel();
}

/// A handoff Complete carries an address snapshotted at handoff creation,
/// delivered on a stream with no ordering against the registration feed.
/// It may only fill a hole, never overwrite: a fresher re-registration
/// must survive a stale handoff-carried address arriving after it.
#[tokio::test]
async fn stale_handoff_address_never_overwrites_a_fresher_registration() {
    let store = test_store("addr-authority").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod = common::start_pod_with_address(
        Arc::clone(&store),
        "writer-0",
        10,
        Some("10.0.0.2:50053".to_string()),
        cancel.clone(),
    );

    // Settle and learn the registered address.
    let addresses = Arc::clone(&router.addresses);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let addresses = Arc::clone(&addresses);
        async move {
            addresses
                .read()
                .unwrap()
                .get("writer-0")
                .map(String::as_str)
                == Some("10.0.0.2:50053")
        }
    })
    .await;

    // A Complete handoff arrives carrying a stale address (snapshotted
    // before the pod's latest registration). It must not clobber.
    store
        .put_handoff(&personhog_coordination::types::HandoffState {
            partition: 0,
            old_owner: None,
            new_owner: "writer-0".to_string(),
            new_owner_address: Some("10.0.0.1:50053".to_string()),
            phase: personhog_coordination::types::HandoffPhase::Complete,
            started_at: 0,
            handoff_id: "h-stale-addr".to_string(),
            freeze_quorum: None,
            freeze_quorum_ref: None,
            created_at_ms: 0,
            phase_entered_at_ms: 0,
            held_by: None,
            config_epoch: 0,
        })
        .await
        .unwrap();

    // The Complete still flows through the routing table (table update +
    // drain); wait for it to be observed, then assert the address held.
    let table = Arc::clone(&router.table);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let table = Arc::clone(&table);
        async move { table.read().await.get(&0).map(String::as_str) == Some("writer-0") }
    })
    .await;
    assert_eq!(
        addresses
            .read()
            .unwrap()
            .get("writer-0")
            .map(String::as_str),
        Some("10.0.0.2:50053"),
        "registration-fed address must survive a stale handoff-carried one"
    );

    cancel.cancel();
}

/// A leader that restarts under the same pod name at a new address while
/// keeping its assignments produces no handoff — the router must learn
/// the new address from the pod registration watch, or every dial keeps
/// hitting the dead address until the next rebalance.
#[tokio::test]
async fn pod_re_registration_refreshes_router_addresses_without_a_handoff() {
    let store = test_store("addr-refresh").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod = common::start_pod_with_address(
        Arc::clone(&store),
        "writer-0",
        10,
        Some("10.0.0.1:50053".to_string()),
        cancel.clone(),
    );

    // Settle: all partitions assigned to writer-0, addresses learned.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;
    let addresses = Arc::clone(&router.addresses);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let addresses = Arc::clone(&addresses);
        async move {
            addresses
                .read()
                .unwrap()
                .get("writer-0")
                .map(String::as_str)
                == Some("10.0.0.1:50053")
        }
    })
    .await;

    // Same pod name re-registers at a new address (a restart that kept
    // its IP-independent identity) — no pod-set change, so no handoff.
    let lease = store.grant_lease(10).await.unwrap();
    let now = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_secs() as i64;
    store
        .register_pod(
            &personhog_coordination::types::RegisteredPod {
                pod_name: "writer-0".to_string(),
                generation: String::new(),
                status: personhog_coordination::types::PodStatus::Ready,
                registered_at: now,
                last_heartbeat: now,
                controller: None,
                advertise_address: Some("10.0.0.2:50053".to_string()),
            },
            lease,
        )
        .await
        .unwrap();

    let addresses = Arc::clone(&router.addresses);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let addresses = Arc::clone(&addresses);
        async move {
            addresses
                .read()
                .unwrap()
                .get("writer-0")
                .map(String::as_str)
                == Some("10.0.0.2:50053")
        }
    })
    .await;

    // The refresh came from the registration watch alone: ownership never
    // moved and the persisted assignments still carry the old snapshot.
    let handoffs = store.list_handoffs().await.unwrap();
    assert!(handoffs.is_empty(), "no handoff may be involved");
    let assignments = store.list_assignments().await.unwrap();
    assert!(
        assignments
            .iter()
            .all(|a| a.owner == "writer-0"
                && a.advertise_address.as_deref() == Some("10.0.0.1:50053"))
    );

    cancel.cancel();
}

/// When the routing table sees a handoff Complete, it should update the
/// table to point at the new owner inline. This proves the inline routing
/// update inside `handle_handoff_put` runs (the assignment watch is gone).
#[tokio::test]
async fn routing_table_updated_at_handoff_complete() {
    let store = test_store("routing-table-inline").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for initial bootstrap
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // Router's table should match etcd
    let table = router.table.read().await;
    assert_eq!(
        table.len(),
        NUM_PARTITIONS as usize,
        "router table should have entries for all partitions"
    );
    assert!(
        table.values().all(|v| v == "writer-0"),
        "router table should point at writer-0"
    );
    drop(table);

    // Add writer-1, triggering handoffs
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());

    // Wait for assignments to be split across both pods AND handoffs to clear
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let has_both = assignments.iter().any(|a| a.owner == "writer-0")
                && assignments.iter().any(|a| a.owner == "writer-1");
            assignments.len() == NUM_PARTITIONS as usize && has_both && handoffs.is_empty()
        }
    })
    .await;

    // Router's table should now reflect the new owner distribution.
    // This is the crucial assertion — without the inline update inside
    // watch_handoffs_loop's Complete handler, the table would not have
    // been updated (we removed watch_assignments_loop).
    let final_assignments = store.list_assignments().await.unwrap();
    let expected: std::collections::HashMap<u32, String> = final_assignments
        .iter()
        .map(|a| (a.partition, a.owner.clone()))
        .collect();
    let table = router.table.read().await;
    assert_eq!(
        *table, expected,
        "router table must match etcd assignments after handoffs complete"
    );

    cancel.cancel();
}

/// A router that joins after a handoff has already entered Warming should
/// observe the Warming event in `load_initial` and call begin_stash. It
/// should NOT write a FreezeAck — the Freezing quorum has already been
/// collected. Verified by injecting a Warming handoff directly into etcd
/// before starting the late-joining router.
#[tokio::test]
async fn late_joining_router_during_warming_begins_stash() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("late-router-warming").await;
    let cancel = CancellationToken::new();

    // Inject a handoff already in Warming — old_owner=writer-0,
    // new_owner=writer-1. This simulates the state a late-joining router
    // sees if it starts up after the freeze quorum is already collected.
    let warming_handoff = HandoffState {
        partition: 3,
        old_owner: Some("writer-0".to_string()),
        new_owner: "writer-1".to_string(),
        phase: HandoffPhase::Warming,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&warming_handoff).await.unwrap();

    // Start the late-joining router
    let router = start_router(Arc::clone(&store), "router-1", cancel.clone());

    // It should observe the Warming handoff via load_initial and begin
    // stashing, despite never having seen a Freezing event for it.
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            events.lock().await.iter().any(|e| {
                matches!(
                    e,
                    common::CutoverEvent::StashBegan { partition: 3, new_owner }
                        if new_owner == "writer-1"
                )
            })
        }
    })
    .await;

    // The router must NOT have written a FreezeAck — the Freezing quorum
    // has already been satisfied (the handoff is past Freezing). Acking
    // here would be incorrect.
    let freeze_acks = store.list_freeze_acks(3).await.unwrap();
    let router1_acks = freeze_acks
        .iter()
        .filter(|a| a.router_name == "router-1")
        .count();
    assert_eq!(
        router1_acks, 0,
        "late-joining router must not write a FreezeAck for a Warming handoff"
    );

    cancel.cancel();
}

/// A Freezing handoff whose `old_owner` is dead (not a registered pod)
/// must still run to completion: the old owner plays no role in Freezing
/// (routers ack the freeze), and Draining treats an absent old owner as
/// vacuously drained. The handoff advances Freezing → Draining → Warming
/// → Complete on the live new owner and is cleaned up by completion, not
/// by cancellation.
#[tokio::test]
async fn dead_old_owner_in_freezing_advances_to_completion() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("dead-old-owner-freezing").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Inject a Freezing handoff: old_owner=phantom-old (never registered
    // as a pod), new_owner=writer-0 (will be registered next).
    let stale = HandoffState {
        partition: 4,
        old_owner: Some("phantom-old".to_string()),
        new_owner: "writer-0".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&stale).await.unwrap();

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // router-0 acks the freeze, the absent old owner drains vacuously,
    // writer-0 warms and acks, and completion deletes the handoff. The
    // deferred rebalance then assigns the remaining partitions.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let assignments = store.list_assignments().await.unwrap_or_default();
            !handoffs
                .iter()
                .any(|h| h.partition == 4 && h.old_owner.as_deref() == Some("phantom-old"))
                && assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "writer-0")
        }
    })
    .await;

    cancel.cancel();
}

/// Symmetric to `late_joining_router_during_warming_begins_stash`, but for a
/// router that comes up while a handoff is still in `Freezing`. The router
/// must both call `begin_stash` AND write a `RouterFreezeAck` — the freeze
/// quorum has not yet been collected, so the router's ack is required for
/// the coordinator to advance.
#[tokio::test]
async fn late_joining_router_during_freezing_acks_and_stashes() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("late-router-freezing").await;
    let cancel = CancellationToken::new();

    // Inject a Freezing handoff before any router is up. This is the state
    // a router sees if it joins after the coordinator created the handoff
    // but before any freeze quorum has been collected.
    let freezing_handoff = HandoffState {
        partition: 5,
        old_owner: Some("writer-0".to_string()),
        new_owner: "writer-1".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&freezing_handoff).await.unwrap();

    // Start the late-joining router.
    let router = start_router(Arc::clone(&store), "router-late", cancel.clone());

    // It must call begin_stash for the partition (observed via load_initial).
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            events.lock().await.iter().any(|e| {
                matches!(
                    e,
                    common::CutoverEvent::StashBegan { partition: 5, new_owner }
                        if new_owner == "writer-1"
                )
            })
        }
    })
    .await;

    // It must also write a FreezeAck — the freeze quorum is still open.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let acks = store.list_freeze_acks(5).await.unwrap_or_default();
            acks.iter().any(|a| a.router_name == "router-late")
        }
    })
    .await;

    cancel.cancel();
}

/// Symmetric to `handoff_delete_drains_stash_to_current_owner`, but with the
/// handoff in `Warming` instead of `Freezing` when it's deleted. Exercises
/// the same `EventType::Delete` branch in `watch_handoffs_loop`, but from
/// the post-freeze state — the router has already begun stashing and the
/// freeze quorum has been collected.
#[tokio::test]
async fn a_dead_new_owner_handoff_in_warming_is_reaffirmed() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("delete-during-warming").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for writer-0 to own all partitions cleanly.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // Inject a Warming handoff for partition 0 targeting a phantom new
    // owner. We use Warming directly so the router observes it as a
    // post-freeze handoff: load_initial calls begin_stash, but no
    // FreezeAck is written.
    let stuck = HandoffState {
        partition: 0,
        old_owner: Some("writer-0".to_string()),
        new_owner: "phantom-pod".to_string(),
        phase: HandoffPhase::Warming,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&stuck).await.unwrap();

    // Wait for the router to observe the handoff and begin stashing.
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            events
                .lock()
                .await
                .iter()
                .any(|e| matches!(e, common::CutoverEvent::StashBegan { partition: 0, .. }))
        }
    })
    .await;

    // Snapshot drain count *before* the Delete. The bootstrap's
    // initial handoffs each completed via the Complete path, which
    // calls drain_stash with target=writer-0 for every partition —
    // including 0. A strict count comparison (not event-existence)
    // is required to attribute the next drain to the Delete event.
    let drains_before = router
        .events
        .lock()
        .await
        .iter()
        .filter(|e| matches!(e, common::CutoverEvent::StashDrained { .. }))
        .count();

    // No deletion is injected: the coordinator detects the dead new
    // owner and cancels by replacement — a reaffirm Complete toward the
    // current owner (writer-0), independent of the phase the doomed
    // handoff was in.

    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            let count = events
                .lock()
                .await
                .iter()
                .filter(|e| matches!(e, common::CutoverEvent::StashDrained { .. }))
                .count();
            count > drains_before
        }
    })
    .await;

    let events = router.events.lock().await;
    let post_delete_drain = events
        .iter()
        .rev()
        .find(|e| matches!(e, common::CutoverEvent::StashDrained { partition: 0, .. }))
        .expect("a Delete-triggered drain for partition 0 must exist");
    match post_delete_drain {
        common::CutoverEvent::StashDrained { target, .. } => {
            assert_eq!(target, "writer-0", "drain must target the current owner");
        }
        _ => unreachable!(),
    }

    cancel.cancel();
}

/// `reconcile_pending_handoffs` must nudge handoffs whose preconditions are
/// already satisfied — the coordinator's ack-watch only fires on Put
/// events, so any acks written before the coordinator came up are invisible
/// to it. Without reconcile, a Warming handoff with a pre-existing
/// `WarmedAck` would stall forever.
///
/// Setup: pre-stage a Warming handoff plus a WarmedAck for its new owner,
/// without a coordinator running. Then start the coordinator and assert
/// the handoff advances to Complete (which deletes the record and writes
/// the new assignment).
#[tokio::test]
async fn reconcile_advances_warming_with_pre_staged_warmed_ack() {
    use personhog_coordination::types::{HandoffPhase, HandoffState, PodWarmedAck};

    let store = test_store("reconcile-pre-staged-ack").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Pre-stage a Warming handoff for partition 6 with no coordinator
    // running. New owner already "warmed" (we inject the ack directly).
    // This is the state the coordinator would face if it crashed right
    // after the new owner wrote its WarmedAck but before the coordinator
    // observed the Put event.
    let warming = HandoffState {
        partition: 6,
        old_owner: Some("writer-0".to_string()),
        new_owner: "writer-1".to_string(),
        phase: HandoffPhase::Warming,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&warming).await.unwrap();
    store
        .put_warmed_ack(&PodWarmedAck {
            pod_name: "writer-1".to_string(),
            partition: 6,
            acked_at: 0,
            acked_at_ms: 0,
            handoff_id: String::new(),
        })
        .await
        .unwrap();

    // Start the coordinator. Reconcile-on-startup must call
    // `check_phase_advance` for the pre-staged handoff, see the existing
    // WarmedAck, and advance Warming → Complete via the atomic txn.
    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    // Once Complete fires, `handle_handoff_update_static` deletes the
    // handoff record and the partition's assignment is written to
    // `writer-1`. Wait for both.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let assignments = store.list_assignments().await.unwrap_or_default();
            let p6_owner = assignments
                .iter()
                .find(|a| a.partition == 6)
                .map(|a| a.owner.as_str());
            !handoffs.iter().any(|h| h.partition == 6) && p6_owner == Some("writer-1")
        }
    })
    .await;

    cancel.cancel();
}

/// A `Draining` pod is still alive (lease present, heartbeating) and is
/// still capable of running its handoff handler and writing a `DrainedAck`.
/// `check_phase_advance` must therefore wait for that ack rather than
/// treating the pod as dead and bypassing the drain. Regression test for
/// the case where gating on `PodStatus::Ready` instead of registration
/// presence let Freezing → Warming advance with potentially in-flight
/// writes still being produced by the old owner.
#[tokio::test]
async fn draining_old_owner_blocks_phase_advance() {
    use personhog_coordination::types::{
        HandoffPhase, HandoffState, PodDrainedAck, PodStatus, RegisteredPod,
    };

    let store = test_store("draining-blocks-advance").await;
    let cancel = CancellationToken::new();

    // Register a Draining pod directly. This simulates a pod that was
    // healthy, started shutting down, and now sits in Draining while
    // working through its remaining handlers.
    let lease = store.grant_lease(30).await.unwrap();
    let draining_pod = RegisteredPod {
        pod_name: "writer-draining".to_string(),
        generation: String::new(),
        status: PodStatus::Draining,
        registered_at: 0,
        last_heartbeat: 0,
        controller: None,
        advertise_address: None,
    };
    store.register_pod(&draining_pod, lease).await.unwrap();

    // Inject a Freezing handoff with the Draining pod as old_owner.
    let handoff = HandoffState {
        partition: 7,
        old_owner: Some("writer-draining".to_string()),
        new_owner: "writer-new".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&handoff).await.unwrap();

    // Start the coordinator. Reconcile-on-startup will call
    // `check_phase_advance`. With no routers registered, the freeze
    // quorum is vacuously satisfied (zero acks needed) so the handoff
    // immediately advances Freezing → Draining. It must then sit in
    // Draining until the (still-alive) Draining pod writes a
    // DrainedAck — without the registration-presence check it would
    // wrongly treat the pod as dead and advance straight to Warming.
    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    // Give the coordinator time to advance Freezing → Draining and to
    // run subsequent ack-watch firings. The handoff must reach Draining
    // and remain there.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            matches!(
                store.get_handoff(7).await.unwrap(),
                Some(ref s) if s.phase == HandoffPhase::Draining
            )
        }
    })
    .await;
    for _ in 0..10 {
        tokio::time::sleep(POLL_INTERVAL).await;
        let h = store.get_handoff(7).await.unwrap();
        assert!(
            matches!(h, Some(ref s) if s.phase == HandoffPhase::Draining),
            "handoff must stay in Draining while old_owner is registered and unacked: {h:?}"
        );
    }

    // Once the Draining pod writes its DrainedAck, the handoff should
    // advance to Warming. The handoff watch's nudge picks up the ack-watch
    // event and re-runs `check_phase_advance`.
    store
        .put_drained_ack(&PodDrainedAck {
            pod_name: "writer-draining".to_string(),
            partition: 7,
            acked_at: 0,
            acked_at_ms: 0,
            handoff_id: String::new(),
        })
        .await
        .unwrap();

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            matches!(
                store.get_handoff(7).await.unwrap(),
                Some(ref s) if s.phase == HandoffPhase::Warming
            )
        }
    })
    .await;

    cancel.cancel();
}

/// The dead-new-owner cancellation trigger judges only the new owner; the
/// old owner's state — Draining, or even fully deregistered — must never
/// cancel a handoff. A `Draining` old owner mid-handoff is the most
/// failure-prone shape: it still owes the protocol a `DrainedAck`, and
/// deleting its handoff record would strand the drain. Regression test
/// for cleanup wrongly keying off old-owner liveness.
#[tokio::test]
async fn draining_old_owner_does_not_trigger_cleanup() {
    use personhog_coordination::types::{HandoffPhase, HandoffState, PodStatus, RegisteredPod};

    let store = test_store("draining-no-cleanup").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Register a Draining pod with no DrainedAck written yet.
    let lease = store.grant_lease(30).await.unwrap();
    let draining_pod = RegisteredPod {
        pod_name: "writer-draining".to_string(),
        generation: String::new(),
        status: PodStatus::Draining,
        registered_at: 0,
        last_heartbeat: 0,
        controller: None,
        advertise_address: None,
    };
    store.register_pod(&draining_pod, lease).await.unwrap();

    // Inject a Freezing handoff with the Draining pod as old_owner.
    let handoff = HandoffState {
        partition: 2,
        old_owner: Some("writer-draining".to_string()),
        new_owner: "writer-new".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&handoff).await.unwrap();

    // Start the coordinator and a real pod. The new pod registering
    // triggers a pod-change event and runs the dead-new-owner cancellation,
    // which must leave the handoff alone: the new owner is live, and the
    // old owner's state is not cleanup's concern.
    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let _new_pod = start_pod(Arc::clone(&store), "writer-new", cancel.clone());

    // Give pod-change handling time to run several times.
    for _ in 0..10 {
        tokio::time::sleep(POLL_INTERVAL).await;
        let h = store.get_handoff(2).await.unwrap();
        assert!(
            h.is_some(),
            "handoff for Draining old_owner must not be deleted by cleanup"
        );
    }

    cancel.cancel();
}

/// The two-phase split (Freezing → Draining → Warming) ensures the old
/// owner only begins draining inflight handlers AFTER every router has
/// freeze-acked. Without the split, a slow router could send a final
/// request to the old owner after the old owner observed inflight=0 and
/// wrote DrainedAck, advancing Kafka HWM past the point warming
/// snapshots and leaving the new owner with a stale cache.
///
/// A handoff whose quorum can never be met must not sit in etcd
/// forever. Nothing else removes it: cleanup only fires when the *new
/// owner* is gone, and an in-flight handoff pins its partition so no
/// re-plan can replace it — which is exactly how a wedge survives until
/// someone restarts a pod by hand. Past its deadline the coordinator
/// deletes it so a later plan can try again.
#[tokio::test]
async fn a_handoff_that_can_never_reach_quorum_is_cancelled() {
    use personhog_coordination::types::{
        HandoffPhase, HandoffState, PodStatus, RegisteredPod, RegisteredRouter,
    };

    let store = test_store("handoff-deadline-cancels").await;
    let cancel = CancellationToken::new();
    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // A live new owner, so the dead-new-owner cleanup path stays out of
    // this: the handoff must be removed by the deadline alone.
    let pod_lease = store.grant_lease(60).await.unwrap();
    store
        .register_pod(
            &RegisteredPod {
                pod_name: "writer-new".to_string(),
                generation: String::new(),
                status: PodStatus::Ready,
                registered_at: 0,
                last_heartbeat: 0,
                controller: None,
                advertise_address: None,
            },
            pod_lease,
        )
        .await
        .unwrap();

    // A router that is required but never acks — the shape of every
    // wedge seen in production.
    let router_lease = store.grant_lease(60).await.unwrap();
    store
        .register_router(
            &RegisteredRouter {
                router_name: "silent-router".to_string(),
                registered_at: 0,
                last_heartbeat: 0,
            },
            router_lease,
        )
        .await
        .unwrap();

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = common::start_coordinator_with_deadline(
        Arc::clone(&store),
        "coordinator-0",
        10,
        Duration::from_secs(1),
        strategy,
        cancel.clone(),
    );

    store
        .put_handoff(&HandoffState {
            partition: 1,
            old_owner: None,
            new_owner: "writer-new".to_string(),
            new_owner_address: None,
            phase: HandoffPhase::Freezing,
            // A real creation time: the deadline is evaluated against it,
            // and a zero here would mean "unknown age", which is never
            // cancelled.
            started_at: std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_secs() as i64,
            handoff_id: "wedged-handoff".to_string(),
            freeze_quorum: Some(vec!["silent-router".to_string()]),
            freeze_quorum_ref: None,
            created_at_ms: 0,
            phase_entered_at_ms: 0,
            held_by: None,
            config_epoch: 0,
        })
        .await
        .unwrap();

    // Cancellation wakes the planner, which is free to create a fresh
    // handoff for the same partition — that successor (different id) is
    // the retry working as designed, not the wedge persisting. The
    // property under test is only that the unsatisfiable handoff itself
    // does not outlive its deadline.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            !matches!(
                store.get_handoff(1).await.unwrap(),
                Some(ref s) if s.handoff_id == "wedged-handoff"
            )
        }
    })
    .await;

    cancel.cancel();
}

/// This test verifies the gate by injecting a Freezing handoff with a
/// registered router that never acks. The handoff must remain in
/// Freezing — it cannot advance to Draining without the freeze quorum,
/// and the old pod must not start draining yet.
#[tokio::test]
async fn freezing_blocks_until_routers_ack_before_draining() {
    use personhog_coordination::types::{HandoffPhase, HandoffState, PodStatus, RegisteredPod};

    let store = test_store("freezing-gates-draining").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Register both owner pods: the new owner so the dead-new-owner cancellation
    // doesn't delete the injected handoff, the old owner so the scenario
    // matches a live in-flight handoff. They don't run real handlers — we
    // just need their etcd registrations to exist.
    let old_lease = store.grant_lease(60).await.unwrap();
    let new_lease = store.grant_lease(60).await.unwrap();
    for (name, lease) in [("writer-old", old_lease), ("writer-new", new_lease)] {
        let pod = RegisteredPod {
            pod_name: name.to_string(),
            generation: String::new(),
            status: PodStatus::Ready,
            registered_at: 0,
            last_heartbeat: 0,
            controller: None,
            advertise_address: None,
        };
        store.register_pod(&pod, lease).await.unwrap();
    }

    // Register a router that will be slow — never write a FreezeAck —
    // so the freeze quorum cannot complete on its own. We register it
    // by hand because no real router process is started.
    let router_lease = store.grant_lease(60).await.unwrap();
    let slow_router = personhog_coordination::types::RegisteredRouter {
        router_name: "slow-router".to_string(),
        registered_at: 0,
        last_heartbeat: 0,
    };
    store
        .register_router(&slow_router, router_lease)
        .await
        .unwrap();

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    // Inject a Freezing handoff. With the slow router never acking,
    // the coordinator cannot advance.
    let handoff = HandoffState {
        partition: 1,
        old_owner: Some("writer-old".to_string()),
        new_owner: "writer-new".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&handoff).await.unwrap();

    // The handoff must stay in Freezing for as long as the slow router
    // hasn't acked, regardless of any other state. This prevents the
    // race where the old pod could be told to drain while a router is
    // still able to forward writes to it.
    for _ in 0..10 {
        tokio::time::sleep(POLL_INTERVAL).await;
        let h = store.get_handoff(1).await.unwrap();
        assert!(
            matches!(h, Some(ref s) if s.phase == HandoffPhase::Freezing),
            "handoff must stay in Freezing without freeze-ack quorum: {h:?}"
        );
    }

    // Once the router acks, the handoff advances to Draining (waiting
    // on the old owner) — not directly to Warming.
    store
        .put_freeze_ack(&personhog_coordination::types::RouterFreezeAck {
            router_name: "slow-router".to_string(),
            partition: 1,
            acked_at: 0,
            acked_at_ms: 0,
            handoff_id: String::new(),
        })
        .await
        .unwrap();

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            matches!(
                store.get_handoff(1).await.unwrap(),
                Some(ref s) if s.phase == HandoffPhase::Draining
            )
        }
    })
    .await;

    // The advance restarts the phase clock: the injected record carried
    // no stamp, so a nonzero value proves the CAS wrote one — what the
    // phase-duration metrics and the per-phase age gauge depend on.
    let advanced = store.get_handoff(1).await.unwrap().unwrap();
    assert!(
        advanced.phase_entered_at_ms > 0,
        "phase advance must stamp phase_entered_at_ms"
    );

    cancel.cancel();
}

/// Initial assignments — handoffs with `old_owner == None` — have
/// nothing to drain and must skip the Draining phase entirely. They
/// advance straight from Freezing to Warming once router quorum is
/// satisfied. Without this short-circuit, every fresh partition
/// assignment would stall in Draining waiting for a DrainedAck that
/// will never arrive.
#[tokio::test]
async fn initial_assignment_skips_draining_phase() {
    use personhog_coordination::types::{HandoffPhase, HandoffState, PodStatus, RegisteredPod};

    let store = test_store("initial-skips-draining").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Register the new_owner pod so the dead-new-owner cancellation doesn't
    // delete the injected handoff for missing-target reasons.
    let lease = store.grant_lease(60).await.unwrap();
    let new_pod = RegisteredPod {
        pod_name: "writer-new".to_string(),
        generation: String::new(),
        status: PodStatus::Ready,
        registered_at: 0,
        last_heartbeat: 0,
        controller: None,
        advertise_address: None,
    };
    store.register_pod(&new_pod, lease).await.unwrap();

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    // Inject a Freezing handoff with no old_owner. With no routers
    // registered, the freeze quorum is vacuously met — the handoff
    // should advance straight from Freezing to Warming, never visiting
    // Draining.
    let handoff = HandoffState {
        partition: 0,
        old_owner: None,
        new_owner: "writer-new".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&handoff).await.unwrap();

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            matches!(
                store.get_handoff(0).await.unwrap(),
                Some(ref s) if s.phase == HandoffPhase::Warming
            )
        }
    })
    .await;

    cancel.cancel();
}

/// If the old owner dies *during* Draining (lease expiry, pod crash),
/// the handoff must still progress: `check_phase_advance`'s "old_owner
/// not registered" branch treats the drain as vacuously complete, and
/// the reconcile tick guarantees re-evaluation even if no watch event
/// fires. The partition must end up assigned to a healthy pod, not
/// stuck in Draining indefinitely.
#[tokio::test]
async fn dead_old_owner_in_draining_recovers() {
    use personhog_coordination::types::{HandoffPhase, HandoffState, PodStatus, RegisteredPod};

    let store = test_store("dead-old-owner-draining").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Register both owners so the injected handoff starts as a live
    // in-flight Draining handoff waiting on a real DrainedAck.
    let old_lease = store.grant_lease(60).await.unwrap();
    let new_lease = store.grant_lease(60).await.unwrap();
    for (name, lease) in [("writer-old", old_lease), ("writer-new", new_lease)] {
        let pod = RegisteredPod {
            pod_name: name.to_string(),
            generation: String::new(),
            status: PodStatus::Ready,
            registered_at: 0,
            last_heartbeat: 0,
            controller: None,
            advertise_address: None,
        };
        store.register_pod(&pod, lease).await.unwrap();
    }

    // Inject a Draining handoff (past Freezing, no DrainedAck yet).
    let handoff = HandoffState {
        partition: 3,
        old_owner: Some("writer-old".to_string()),
        new_owner: "writer-new".to_string(),
        phase: HandoffPhase::Draining,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&handoff).await.unwrap();

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    // Wait for the coordinator to settle — the handoff must be in
    // Draining (no DrainedAck written, old_owner still registered).
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            matches!(
                store.get_handoff(3).await.unwrap(),
                Some(ref s) if s.phase == HandoffPhase::Draining
            )
        }
    })
    .await;

    // Kill the old owner. The next re-evaluation (pod-change event or
    // reconcile tick) must advance the drain vacuously.
    store.delete_pod("writer-old").await.unwrap();

    // End-state assertion: no Draining handoff lingers for partition
    // 3, and partition 3 ends up assigned to a registered pod.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let p3_still_draining = handoffs
                .iter()
                .any(|h| h.partition == 3 && h.phase == HandoffPhase::Draining);
            if p3_still_draining {
                return false;
            }
            // Partition is either fully resolved (assignment present,
            // handoff complete or absent) or progressing (handoff has
            // moved past Draining toward Warming/Complete).
            let assignments = store.list_assignments().await.unwrap_or_default();
            let p3_assigned = assignments
                .iter()
                .any(|a| a.partition == 3 && a.owner == "writer-new");
            let p3_handoff_progressing = handoffs
                .iter()
                .any(|h| h.partition == 3 && h.phase != HandoffPhase::Draining);
            p3_assigned || p3_handoff_progressing
        }
    })
    .await;

    cancel.cancel();
}

/// Reconcile-on-startup must be able to advance a Draining handoff
/// whose preconditions were satisfied before the coordinator came up.
/// Two pre-stages exercise this: a DrainedAck already in etcd, or the
/// old_owner already absent. Either way the coordinator's reconcile
/// pass should drive the handoff to Warming via `check_phase_advance`,
/// since the ack-watch only fires on Put events and missed pre-existing
/// state.
#[tokio::test]
async fn reconcile_advances_draining_with_pre_staged_drained_ack() {
    use personhog_coordination::types::{
        HandoffPhase, HandoffState, PodDrainedAck, PodStatus, RegisteredPod,
    };

    let store = test_store("reconcile-draining-ack").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    // Pre-stage both pods (so cleanup doesn't fire on either) plus a
    // Draining handoff plus a matching DrainedAck — all before the
    // coordinator starts. Reconcile must observe this state and
    // advance Draining → Warming.
    for name in ["writer-old", "writer-new"] {
        let lease = store.grant_lease(60).await.unwrap();
        let pod = RegisteredPod {
            pod_name: name.to_string(),
            generation: String::new(),
            status: PodStatus::Ready,
            registered_at: 0,
            last_heartbeat: 0,
            controller: None,
            advertise_address: None,
        };
        store.register_pod(&pod, lease).await.unwrap();
    }

    let handoff = HandoffState {
        partition: 5,
        old_owner: Some("writer-old".to_string()),
        new_owner: "writer-new".to_string(),
        phase: HandoffPhase::Draining,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&handoff).await.unwrap();

    store
        .put_drained_ack(&PodDrainedAck {
            pod_name: "writer-old".to_string(),
            partition: 5,
            acked_at: 0,
            acked_at_ms: 0,
            handoff_id: String::new(),
        })
        .await
        .unwrap();

    // Now start the coordinator. Without reconcile-driven advancement,
    // the pre-staged DrainedAck would be invisible (the ack-watch only
    // fires on Put events delivered after the watch is established) and
    // the handoff would stall. Reconcile re-runs `check_phase_advance`
    // for every existing handoff, which picks up the pre-staged ack and
    // advances to Warming.
    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            matches!(
                store.get_handoff(5).await.unwrap(),
                Some(ref s) if s.phase == HandoffPhase::Warming
            )
        }
    })
    .await;

    cancel.cancel();
}

/// Symmetric to `late_joining_router_during_warming_begins_stash`, but
/// for a router that comes up while the handoff is in `Draining`. The
/// router must call `begin_stash` (so future writes for the partition
/// are buffered) and must NOT write a `FreezeAck` — the freeze quorum
/// has already been collected, and a late ack could miscount toward a
/// future handoff for the same partition.
#[tokio::test]
async fn late_joining_router_during_draining_begins_stash_no_ack() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("late-router-draining").await;
    let cancel = CancellationToken::new();

    // Inject a Draining handoff before any router is up.
    let draining_handoff = HandoffState {
        partition: 2,
        old_owner: Some("writer-0".to_string()),
        new_owner: "writer-1".to_string(),
        phase: HandoffPhase::Draining,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&draining_handoff).await.unwrap();

    // Start the late-joining router.
    let router = start_router(Arc::clone(&store), "router-late", cancel.clone());

    // It must call begin_stash for the partition (observed via
    // load_initial).
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            events.lock().await.iter().any(|e| {
                matches!(
                    e,
                    common::CutoverEvent::StashBegan { partition: 2, new_owner }
                        if new_owner == "writer-1"
                )
            })
        }
    })
    .await;

    // The router must NOT have written a FreezeAck — the freeze
    // quorum closed when the coordinator advanced Freezing → Draining,
    // and a late ack would either be redundant or risk being counted
    // toward the next handoff for this partition.
    tokio::time::sleep(POLL_INTERVAL * 3).await;
    let freeze_acks = store.list_freeze_acks(2).await.unwrap();
    let router1_acks = freeze_acks
        .iter()
        .filter(|a| a.router_name == "router-late")
        .count();
    assert_eq!(
        router1_acks, 0,
        "late-joining router must not write a FreezeAck for a Draining handoff"
    );

    cancel.cancel();
}

/// Symmetric to `handoff_delete_drains_stash_to_current_owner` and
/// `handoff_delete_during_warming_drains_to_current_owner`, but for a
/// handoff in `Draining` when it's deleted. The Delete branch in
/// `watch_handoffs_loop` is phase-agnostic, so this exercises the same
/// code path; the test confirms the recovery is uniform across phases.
#[tokio::test]
async fn a_dead_new_owner_handoff_in_draining_is_reaffirmed() {
    use personhog_coordination::types::{HandoffPhase, HandoffState};

    let store = test_store("delete-during-draining").await;
    let cancel = CancellationToken::new();

    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
    let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
    let router = start_router(Arc::clone(&store), "router-0", cancel.clone());
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    // Wait for the bootstrap assignment to settle so the router's
    // table points at writer-0 for every partition.
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // Inject a Draining handoff for partition 0 targeting a phantom
    // new owner. Inserting at Draining (rather than Freezing) lets us
    // exercise the Delete-during-Draining branch directly.
    let stuck = HandoffState {
        partition: 0,
        old_owner: Some("writer-0".to_string()),
        new_owner: "phantom-pod".to_string(),
        phase: HandoffPhase::Draining,
        started_at: 0,
        handoff_id: String::new(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    store.put_handoff(&stuck).await.unwrap();

    // Wait for the router to observe the handoff and begin stashing.
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            events
                .lock()
                .await
                .iter()
                .any(|e| matches!(e, common::CutoverEvent::StashBegan { partition: 0, .. }))
        }
    })
    .await;

    // Snapshot drain count before the Delete (see the Freezing
    // variant's comment for why a strict count, not event-existence,
    // is required).
    let drains_before = router
        .events
        .lock()
        .await
        .iter()
        .filter(|e| matches!(e, common::CutoverEvent::StashDrained { .. }))
        .count();

    // No deletion is injected: the coordinator detects the dead new
    // owner and cancels by replacement — a reaffirm Complete toward the
    // current owner (writer-0), independent of the phase the doomed
    // handoff was in.

    // Wait for the drain *count* to grow past `drains_before`.
    // Bootstrap drains for partition 0 already exist (each initial
    // handoff completes via Complete and calls drain_stash with
    // target=writer-0), so an event-existence predicate would race
    // and return true before the Delete-triggered drain fires.
    let check_router = Arc::clone(&router.events);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let events = Arc::clone(&check_router);
        async move {
            let count = events
                .lock()
                .await
                .iter()
                .filter(|e| matches!(e, common::CutoverEvent::StashDrained { .. }))
                .count();
            count > drains_before
        }
    })
    .await;

    let events = router.events.lock().await;
    let post_delete_drain = events
        .iter()
        .rev()
        .find(|e| matches!(e, common::CutoverEvent::StashDrained { partition: 0, .. }))
        .expect("a Delete-triggered drain for partition 0 must exist");
    match post_delete_drain {
        common::CutoverEvent::StashDrained { target, .. } => {
            assert_eq!(target, "writer-0", "drain must target the current owner");
        }
        _ => unreachable!(),
    }

    cancel.cancel();
}

// ============================================================
// Graceful exits release etcd state immediately, never by TTL
// ============================================================

/// A cancelled coordinator must revoke its election lease on the way out.
/// With a 30s TTL, a pass that relied on lease expiry would blow the 3s
/// bound — this pins the awaited-to-completion cleanup in
/// `Coordinator::run`, which an unbiased select against cancellation used
/// to drop about half the time, stalling every handoff until the TTL ran
/// out.
#[tokio::test]
async fn graceful_coordinator_exit_releases_election_immediately() {
    let store = test_store("graceful-coordinator-release").await;
    let cancel = CancellationToken::new();
    let handle = start_coordinator_named(
        Arc::clone(&store),
        "coordinator-0",
        30,
        Arc::new(StickyBalancedStrategy),
        cancel.clone(),
    );

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.get_leader().await.ok().flatten().is_some() }
    })
    .await;

    cancel.cancel();
    drop(handle.await);

    let check_store = Arc::clone(&store);
    wait_for_condition(Duration::from_secs(3), POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.get_leader().await.ok().flatten().is_none() }
    })
    .await;
}

/// A cancelled routing table must deregister its router on the way out: a
/// lingering registration keeps the dead router in every freeze quorum,
/// stalling any handoff frozen inside the TTL window.
#[tokio::test]
async fn routing_table_deregisters_router_on_graceful_exit() {
    let store = test_store("graceful-router-deregister").await;
    let cancel = CancellationToken::new();
    let mut router =
        start_router_with_lease_ttl(Arc::clone(&store), "router-0", 30, cancel.clone());

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { !store.list_routers().await.unwrap_or_default().is_empty() }
    })
    .await;

    cancel.cancel();
    if let Some(handle) = router.join_handle.take() {
        drop(handle.await);
    }

    let check_store = Arc::clone(&store);
    wait_for_condition(Duration::from_secs(3), POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_routers().await.unwrap_or_default().is_empty() }
    })
    .await;
}

/// A stuck handoff must defer only its own partition. One pod's handoffs
/// wedge at Warming (its handler never acks); a healthy pod that joins
/// afterwards must still receive partitions from the unpinned remainder
/// while the wedged handoffs stay in flight — and those handoffs must
/// never be re-planned. Before per-partition pinning, any in-flight
/// handoff deferred all rebalancing, so the healthy pod owned nothing
/// until the wedge resolved.
#[tokio::test]
async fn stuck_handoff_defers_only_its_own_partition() {
    let store = test_store("stuck_handoff_defers_only_its_own_partition").await;
    let cancel = CancellationToken::new();
    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::new(StickyBalancedStrategy),
        cancel.clone(),
    );
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    // The blocking pod joins: the rebalance moves partitions toward it and
    // every one of those handoffs wedges at Warming.
    let _blocked = start_pod_blocking(Arc::clone(&store), "writer-blocked", 30, cancel.clone());
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            store
                .list_handoffs()
                .await
                .unwrap_or_default()
                .iter()
                .any(|h| h.new_owner == "writer-blocked")
        }
    })
    .await;

    let stuck: Vec<(u32, String)> = store
        .list_handoffs()
        .await
        .unwrap()
        .into_iter()
        .filter(|h| h.new_owner == "writer-blocked")
        .map(|h| (h.partition, h.handoff_id))
        .collect();
    assert!(!stuck.is_empty());

    // A healthy pod joins while those handoffs are wedged: it must own
    // partitions while they are still in flight.
    let _pod2 = start_pod(Arc::clone(&store), "writer-2", cancel.clone());
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.iter().any(|a| a.owner == "writer-2")
                && handoffs.iter().any(|h| h.new_owner == "writer-blocked")
        }
    })
    .await;

    // The wedged handoffs were pinned, never re-planned: each survives
    // with its original identity.
    let handoffs = store.list_handoffs().await.unwrap();
    for (partition, handoff_id) in &stuck {
        assert!(
            handoffs
                .iter()
                .any(|h| h.partition == *partition && h.handoff_id == *handoff_id),
            "pinned handoff for partition {partition} was replanned or destroyed"
        );
    }
}

/// Concurrent planners can both read a partition as unpinned and plan it;
/// handoff creation is guarded create-if-absent so the second plan
/// cannot replace the first handoff and orphan its acks. Atomicity is
/// per partition: the losing plan's uncontested writes still land, and
/// only the contested partition stands down.
#[tokio::test]
async fn conflicting_plan_cannot_replace_an_in_flight_handoff() {
    use personhog_coordination::types::{
        AssignmentStatus, HandoffPhase, HandoffState, PartitionAssignment,
    };

    let store = test_store("conflicting_plan_cannot_replace_an_in_flight_handoff").await;

    let first = HandoffState {
        partition: 0,
        old_owner: Some("writer-0".to_string()),
        new_owner: "writer-1".to_string(),
        phase: HandoffPhase::Warming,
        started_at: 0,
        handoff_id: "first".to_string(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    assert!(store
        .create_assignments_and_handoffs(&[], std::slice::from_ref(&first), &[], 128)
        .await
        .unwrap());

    // A competing plan targets the same partition (different owner and
    // id) alongside an uncontested assignment write.
    let competing = HandoffState {
        partition: 0,
        old_owner: Some("writer-0".to_string()),
        new_owner: "writer-2".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: "second".to_string(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };
    let stable = PartitionAssignment {
        partition: 1,
        owner: "writer-0".to_string(),
        status: AssignmentStatus::Active,
        advertise_address: None,
    };
    assert!(!store
        .create_assignments_and_handoffs(&[stable], &[competing], &[], 128)
        .await
        .unwrap());

    // The in-flight handoff survives with its original identity; the
    // losing plan's uncontested assignment landed on its own.
    let handoffs = store.list_handoffs().await.unwrap();
    assert_eq!(handoffs.len(), 1);
    assert_eq!(handoffs[0].handoff_id, "first");
    assert_eq!(handoffs[0].new_owner, "writer-1");
    let assignments = store.list_assignments().await.unwrap();
    assert_eq!(assignments.len(), 1);
    assert_eq!(assignments[0].partition, 1);
}

/// The reviewer's stale-plan scenario at the store level: a plan whose
/// snapshot predates a concurrent move (create → complete → cleanup, so
/// the handoff key is absent again) must be rejected by its assignment
/// precondition — its handoff names a superseded old_owner, and applying
/// it would drain the wrong pod while the real owner stays unfenced.
#[tokio::test]
async fn stale_plan_is_rejected_when_the_assignment_moved() {
    use personhog_coordination::types::{
        AssignmentPrecondition, AssignmentStatus, HandoffPhase, HandoffState, PartitionAssignment,
    };

    let store = test_store("stale_plan_is_rejected_when_the_assignment_moved").await;

    let assignment = |owner: &str| PartitionAssignment {
        partition: 0,
        owner: owner.to_string(),
        status: AssignmentStatus::Active,
        advertise_address: None,
    };
    let handoff = |old: &str, new: &str, id: &str| HandoffState {
        partition: 0,
        old_owner: Some(old.to_string()),
        new_owner: new.to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: id.to_string(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };

    // The world the stale planner reads: partition 0 owned by A.
    assert!(store
        .create_assignments_and_handoffs(&[assignment("pod-a")], &[], &[], 128)
        .await
        .unwrap());
    let snapshot = store.list_assignments_with_mod_revisions().await.unwrap();
    let (_, stale_revision) = snapshot[0];

    // Concurrently, a full move to B lands: by the time the stale plan's
    // txn arrives, the handoff key is gone and only the assignment moved.
    assert!(store
        .create_assignments_and_handoffs(&[assignment("pod-b")], &[], &[], 128)
        .await
        .unwrap());

    // The stale plan (move A -> C, guarded by its snapshot) must fail
    // whole, creating nothing.
    let rejected = store
        .create_assignments_and_handoffs(
            &[],
            &[handoff("pod-a", "pod-c", "stale")],
            &[AssignmentPrecondition::UnchangedSince {
                partition: 0,
                mod_revision: stale_revision,
            }],
            128,
        )
        .await
        .unwrap();
    assert!(
        !rejected,
        "a plan reading a superseded owner must not apply"
    );
    assert!(store.list_handoffs().await.unwrap().is_empty());

    // A plan computed from the current snapshot applies fine.
    let snapshot = store.list_assignments_with_mod_revisions().await.unwrap();
    let (current, fresh_revision) = &snapshot[0];
    assert_eq!(current.owner, "pod-b");
    assert!(store
        .create_assignments_and_handoffs(
            &[],
            &[handoff("pod-b", "pod-c", "fresh")],
            &[AssignmentPrecondition::UnchangedSince {
                partition: 0,
                mod_revision: *fresh_revision,
            }],
            128
        )
        .await
        .unwrap());
    let handoffs = store.list_handoffs().await.unwrap();
    assert_eq!(handoffs.len(), 1);
    assert_eq!(handoffs[0].old_owner.as_deref(), Some("pod-b"));
}

/// A fresh-partition handoff asserts the assignment is still absent: if
/// one appeared since the snapshot (the partition got born through a
/// concurrent handoff's completion), the plan's old_owner: None is a lie
/// and that partition must be rejected — alone: a stale precondition
/// stands down its own partition, not the plan's valid handoffs.
#[tokio::test]
async fn fresh_plan_is_rejected_when_an_assignment_appeared() {
    use personhog_coordination::types::{
        AssignmentPrecondition, AssignmentStatus, HandoffPhase, HandoffState, PartitionAssignment,
    };

    let store = test_store("fresh_plan_is_rejected_when_an_assignment_appeared").await;

    let fresh_handoff = |partition: u32, id: &str| HandoffState {
        partition,
        old_owner: None,
        new_owner: "pod-c".to_string(),
        phase: HandoffPhase::Freezing,
        started_at: 0,
        handoff_id: id.to_string(),
        freeze_quorum: None,
        freeze_quorum_ref: None,
        created_at_ms: 0,
        phase_entered_at_ms: 0,
        held_by: None,
        config_epoch: 0,
        new_owner_address: None,
    };

    // Partition 0 gained an assignment after the plan's snapshot.
    assert!(store
        .create_assignments_and_handoffs(
            &[PartitionAssignment {
                partition: 0,
                owner: "pod-b".to_string(),
                status: AssignmentStatus::Active,
                advertise_address: None,
            }],
            &[],
            &[],
            128
        )
        .await
        .unwrap());

    // The plan carries one stale fresh-handoff (partition 0) and one
    // genuinely fresh one (partition 1): the stale partition stands
    // down alone, the innocent one applies.
    let all_applied = store
        .create_assignments_and_handoffs(
            &[],
            &[fresh_handoff(0, "stale"), fresh_handoff(1, "innocent")],
            &[
                AssignmentPrecondition::Absent { partition: 0 },
                AssignmentPrecondition::Absent { partition: 1 },
            ],
            128,
        )
        .await
        .unwrap();
    assert!(!all_applied);
    let handoffs = store.list_handoffs().await.unwrap();
    assert_eq!(handoffs.len(), 1);
    assert_eq!(handoffs[0].partition, 1);
    assert_eq!(handoffs[0].handoff_id, "innocent");
}

/// The rebalance must never write assignment records — handoff completion
/// is their sole writer, so routers always observe owner changes as
/// Complete events and a stale plan can never restore a superseded owner.
/// Detected via mod revisions, since a redundant rewrite is byte-identical.
#[tokio::test]
async fn rebalance_never_writes_assignment_records() {
    let store = test_store("rebalance_never_writes_assignment_records").await;
    let cancel = CancellationToken::new();
    assert!(store
        .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
        .await
        .unwrap());

    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::new(StickyBalancedStrategy),
        cancel.clone(),
    );
    let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize && handoffs.is_empty()
        }
    })
    .await;

    let before: std::collections::HashMap<u32, i64> = store
        .list_assignments_with_mod_revisions()
        .await
        .unwrap()
        .into_iter()
        .map(|(a, revision)| (a.partition, revision))
        .collect();

    // A second pod joins: a rebalance runs and its handoffs complete.
    let _pod1 = start_pod(Arc::clone(&store), "writer-1", cancel.clone());
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            handoffs.is_empty() && assignments.iter().any(|a| a.owner == "writer-1")
        }
    })
    .await;

    let after = store.list_assignments_with_mod_revisions().await.unwrap();
    for (assignment, revision) in &after {
        if assignment.owner == "writer-0" {
            assert_eq!(
                before[&assignment.partition], *revision,
                "partition {} was not moved, yet its assignment record was rewritten",
                assignment.partition
            );
        } else {
            assert_ne!(
                before[&assignment.partition], *revision,
                "partition {} moved, so completion must have rewritten its record",
                assignment.partition
            );
        }
    }
}

/// The store stamps `acked_at_ms` at put time so span metrics can measure
/// ack-to-advance lag; writers all pass zero and must get a real stamp back.
#[tokio::test]
async fn store_stamps_ack_millis_on_put() {
    let store = test_store("ack-ms-stamp").await;
    store
        .put_freeze_ack(&personhog_coordination::types::RouterFreezeAck {
            router_name: "router-0".to_string(),
            partition: 3,
            acked_at: 1_700_000_000,
            acked_at_ms: 0,
            handoff_id: "h-1".to_string(),
        })
        .await
        .expect("put freeze ack");
    let acks = store.list_freeze_acks(3).await.expect("list freeze acks");
    assert_eq!(acks.len(), 1);
    assert!(
        acks[0].acked_at_ms > 0,
        "store must stamp the millisecond clock on ack writes"
    );
}