Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

145 KB
Raw Blame
//! The `cohort_stream_events` wire envelope and the group consumer that drives it.
//!
//! [`CohortStreamEvent`] is defined in `cohort-core` and intentionally shared with the backfill
//! seeder, which produces the same envelope. It stays decoupled from `cohort-event-shuffler`'s
//! producer type — they share only the JSON field names, so neither can break the other by adding a
//! one-sided field.
//!
//! The Kafka-free routing core lives in [`EventDispatcher`] so it can be unit-tested with an
//! in-process router/store/catalog and no broker.

use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};

use chrono::Utc;
use cohort_core::seed::CoverageStartMs;
use dashmap::mapref::entry::Entry;
use dashmap::DashMap;
use lifecycle::Handle;
use metrics::{counter, gauge, histogram};
use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};
use rdkafka::error::KafkaResult;
use rdkafka::message::Message;
use rdkafka::Offset;
use tracing::{debug, error, info, warn};

pub use cohort_core::events::CohortStreamEvent;

use crate::consumers::boot::{BootPhase, ResumePoints};
use crate::consumers::merges::{ConsumedCascade, ConsumedMerge, ConsumedTransfer};
use crate::consumers::readiness::BootReadiness;
use crate::consumers::seeds::ConsumedSeed;
use crate::filters::manager::CatalogHandle;
use crate::merge::transfer::PendingTransfer;
use crate::observability::metrics::{
    CHECKPOINT_RESTORE_BLOCKED, COHORT_STREAM_CASCADES_SKIPPED_NOT_OWNED,
    COHORT_STREAM_CONSUME_BATCH_SIZE, COHORT_STREAM_DESERIALIZE_ERRORS,
    COHORT_STREAM_EMPTY_PAYLOAD, COHORT_STREAM_EVENTS_CONSUMED, COHORT_STREAM_EVENTS_DISPATCHED,
    COHORT_STREAM_EVENTS_SKIPPED_NOT_OWNED, COHORT_STREAM_KAFKA_RECV_ERRORS,
    COHORT_STREAM_MERGES_SKIPPED_NOT_OWNED, COHORT_STREAM_OFFSET_COMMITS,
    COHORT_STREAM_OFFSET_COMMIT_ERRORS, COHORT_STREAM_ROUTE_ERRORS,
    COHORT_STREAM_SEEDS_SKIPPED_NOT_OWNED, COHORT_STREAM_TRANSFERS_SKIPPED_NOT_OWNED,
    COHORT_STREAM_WORKERS_SPAWNED, DURABLE_RESTORE_PARTITIONS_KEPT_TOTAL,
    DURABLE_RESTORE_PARTITIONS_WIPED_STALE_TOTAL,
    DURABLE_RESTORE_PENDING_TRANSFERS_RECOVERED_PARTITIONS_TOTAL, OWNED_PARTITIONS,
    PARTITIONS_ASSIGNED_TOTAL, PARTITIONS_PAUSED, PARTITION_OWNERSHIP_INFO, PENDING_HELD_EVENTS,
};
use crate::partitions::backpressure::Backpressure;
use crate::partitions::offset_tracker::OffsetTracker;
use crate::partitions::pause::{ConsumerPauser, PartitionPauser};
use crate::partitions::rebalance::CohortConsumerContext;
use crate::partitions::router::{PartitionRouter, SeedRefusal, SeedSendOutcome, SendOutcome};
use crate::partitions::shuffle_message::ShuffleMessage;
use crate::partitions::{list_offsets, ClaimedSplit, CommitStamp, InputTopic, PartitionOwnership};
use crate::producer::MembershipSink;
use crate::store::durability::{FinishError, PendingRestore};
use crate::store::StoreHandle;
use crate::workers::{EventNameGating, EvictionRestore, MergeWorkerDeps, Stage1Worker};

/// Back-off after a Kafka transport error so a fast-failing `recv()` can't spin a consume loop.
pub(crate) const RECV_ERROR_BACKOFF: Duration = Duration::from_millis(500);

/// Timeout for the boot seek of the events topic. A local fetch-position reposition, so a few
/// seconds is ample; a timeout is a seek failure and retries (never resumes from the broker offset).
const BOOT_SEEK_TIMEOUT: Duration = Duration::from_secs(10);

/// Page size for the eager boot redrive's paginated scan of `cf_pending_transfers`. The outbox only
/// ever holds transfers stranded by inline-retry exhaustion (rare), so this large page drains a
/// typical outbox in a single round; a larger backlog is drained across successive pages, bounding
/// peak memory per page.
const EAGER_BOOT_REDRIVE_PAGE_SIZE: usize = 100_000;

/// One event consumed from `cohort_stream_events`, paired with its commit coordinates on that topic.
///
/// `partition`/`offset` here are commit coordinates, distinct from the event's upstream
/// `source_partition`/`source_offset` which drive per-key replay idempotence.
#[derive(Debug)]
pub struct ConsumedEvent {
    pub event: CohortStreamEvent,
    pub partition: i32,
    pub offset: i64,
    /// Broker timestamp (CreateTime ≈ shuffler produce instant) — the seed fence's live-watermark
    /// input. `None` (`Timestamp::NotAvailable`) never advances the watermark.
    pub broker_ts_ms: Option<i64>,
}

/// The partitions this pod owned when its workers finished draining. Followers make their final
/// commits for this snapshot. Only [`EventDispatcher::shutdown`] builds one.
#[derive(Debug, Clone)]
pub(crate) struct DrainedOwnership(HashSet<i32>);

impl DrainedOwnership {
    pub(crate) fn partitions(&self) -> &HashSet<i32> {
        &self.0
    }

    #[cfg(test)]
    pub(crate) fn for_test(partitions: impl IntoIterator<Item = i32>) -> Self {
        Self(partitions.into_iter().collect())
    }
}

pub(crate) enum DrainWait {
    Drained(DrainedOwnership),
    TimedOut,
}

/// The Kafka-free routing core: dispatches consumed batches to per-partition workers, tracks
/// processed offsets, and records which partitions this pod owns.
pub struct EventDispatcher {
    router: Arc<PartitionRouter>,
    tracker: Arc<OffsetTracker>,
    workers: Arc<DashMap<i32, Stage1Worker>>,
    /// Partitions this pod owns, with when each was assigned. A slice without a coverage record
    /// begins at that instant, not at its worker's spawn.
    owned: Arc<DashMap<i32, CoverageStartMs>>,
    handle: StoreHandle,
    catalog: Arc<CatalogHandle>,
    sink: Arc<dyn MembershipSink>,
    merge: Arc<MergeWorkerDeps>,
    /// Prevents post-shutdown worker registration that would hang the join. The structural guarantee
    /// is the router's terminal closed state, which refuses registration after `clear()`.
    draining: AtomicBool,
    /// When set, workers rebuild their `EvictionQueue` on spawn and boot reclaims the on-disk
    /// slices of partitions this pod does not own. Set once at startup before any worker spawns.
    durable_restore: AtomicBool,
    /// Event-name fan-out gating for spawned workers, set once at startup. Unset → disabled.
    event_name_gating: OnceLock<EventNameGating>,
    /// `/_ready`'s gate. Every rebuilding worker this dispatcher spawns holds it closed.
    readiness: Arc<BootReadiness>,
    /// Set once the shutdown drain ends, for the followers' final commits.
    drained: tokio::sync::SetOnce<DrainedOwnership>,
}

impl EventDispatcher {
    pub fn new(
        router: PartitionRouter,
        tracker: Arc<OffsetTracker>,
        handle: StoreHandle,
        catalog: Arc<CatalogHandle>,
        sink: Arc<dyn MembershipSink>,
        merge: Arc<MergeWorkerDeps>,
    ) -> Self {
        Self {
            router: Arc::new(router),
            tracker,
            workers: Arc::new(DashMap::new()),
            owned: Arc::new(DashMap::new()),
            handle,
            readiness: BootReadiness::new(catalog.clone()),
            catalog,
            sink,
            merge,
            draining: AtomicBool::new(false),
            durable_restore: AtomicBool::new(false),
            event_name_gating: OnceLock::new(),
            drained: tokio::sync::SetOnce::new(),
        }
    }

    pub fn readiness(&self) -> &Arc<BootReadiness> {
        &self.readiness
    }

    /// Gate `/_ready` on `readiness`, which a health server bound before the dispatcher already
    /// serves.
    pub fn with_readiness(mut self, readiness: Arc<BootReadiness>) -> Self {
        self.readiness = readiness;
        self
    }

    /// Enable crash-restart durability. Must be called before any worker spawns.
    pub fn enable_durable_restore(&self) {
        self.durable_restore.store(true, Ordering::SeqCst);
    }

    pub fn durable_restore_enabled(&self) -> bool {
        self.durable_restore.load(Ordering::SeqCst)
    }

    /// Must be called before any worker spawns; later calls are ignored.
    pub fn set_event_name_gating(&self, gating: EventNameGating) {
        let _ = self.event_name_gating.set(gating);
    }

    fn event_name_gating(&self) -> EventNameGating {
        self.event_name_gating
            .get()
            .copied()
            .unwrap_or(EventNameGating::Disabled)
    }

    pub(crate) fn handle(&self) -> &StoreHandle {
        &self.handle
    }

    /// Blocking events dispatch, retained for tests; the consume loop uses
    /// [`dispatch_events_nonblocking`](Self::dispatch_events_nonblocking).
    ///
    /// The dispatch ceiling is raised before routing so a `RouteError` leaves the offset
    /// uncommittable and Kafka replays it. Events for unowned partitions are dropped before any
    /// ceiling bump.
    pub async fn dispatch(&self, batch: Vec<ConsumedEvent>) {
        if batch.is_empty() {
            return;
        }
        let items = batch
            .into_iter()
            .map(|consumed| {
                (
                    consumed.partition,
                    consumed.offset,
                    ShuffleMessage::Event {
                        event: Box::new(consumed.event),
                        cse_offset: consumed.offset,
                        broker_ts_ms: consumed.broker_ts_ms,
                    },
                )
            })
            .collect();
        let stats = self.dispatch_to_workers(items, &self.tracker).await;
        counter!(COHORT_STREAM_EVENTS_SKIPPED_NOT_OWNED).increment(stats.not_owned_skipped);
        counter!(COHORT_STREAM_EVENTS_DISPATCHED).increment(stats.dispatched);
    }

    /// Non-blocking events dispatch: `try_send`s each non-held partition's sub-batch and raises the
    /// dispatch ceiling only for what lands. Events for a partition in `held` (already paused behind an
    /// older holdover) skip the send and are returned to queue behind it. Returns the un-dispatched
    /// events per partition for the caller to hold and pause.
    #[hotpath::measure(impl_type = "EventDispatcher")]
    pub fn dispatch_events_nonblocking(
        &self,
        batch: Vec<ConsumedEvent>,
        held: &HashSet<i32>,
    ) -> HashMap<i32, Vec<ShuffleMessage>> {
        if self.draining.load(Ordering::SeqCst) {
            if !batch.is_empty() {
                counter!(COHORT_STREAM_EVENTS_SKIPPED_NOT_OWNED).increment(batch.len() as u64);
                debug!(
                    dropped = batch.len(),
                    "dispatch after shutdown began; dropping batch for replay"
                );
            }
            return HashMap::new();
        }

        let mut full: HashMap<i32, Vec<ShuffleMessage>> = HashMap::new();
        let mut fresh: Vec<(i32, ShuffleMessage)> = Vec::with_capacity(batch.len());
        let mut not_owned = 0u64;
        for consumed in batch {
            let partition = consumed.partition;
            if !self.owned.contains_key(&partition) {
                not_owned += 1;
                continue;
            }
            self.ensure_worker(partition);
            let message = ShuffleMessage::Event {
                event: Box::new(consumed.event),
                cse_offset: consumed.offset,
                broker_ts_ms: consumed.broker_ts_ms,
            };
            // A paused partition can still deliver a few already-fetched stragglers; queue them behind
            // the holdover rather than racing them ahead of older offsets.
            if held.contains(&partition) {
                full.entry(partition).or_default().push(message);
            } else {
                fresh.push((partition, message));
            }
        }
        counter!(COHORT_STREAM_EVENTS_SKIPPED_NOT_OWNED).increment(not_owned);

        self.route_and_fold(fresh, &mut full);
        full
    }

    /// Retry-send held sub-batches, raising the ceiling for any that flush. Returns the partitions
    /// still unavailable (full, missing, or closed) to keep paused; only a flushed partition is
    /// absent, so the caller resumes it.
    pub fn redispatch_held(
        &self,
        held: Vec<(i32, Vec<ShuffleMessage>)>,
    ) -> HashMap<i32, Vec<ShuffleMessage>> {
        // Each partition's Vec stays contiguous, so the regrouped route keeps offset order.
        let mut messages: Vec<(i32, ShuffleMessage)> = Vec::new();
        for (partition, batch) in held {
            self.ensure_worker(partition);
            for message in batch {
                messages.push((partition, message));
            }
        }
        let mut still_full = HashMap::new();
        self.route_and_fold(messages, &mut still_full);
        still_full
    }

    /// `try_send` events and fold the outcomes: raise the ceiling for what landed, collect a full
    /// channel's un-sent sub-batch into `full`, and count a missing/closed worker as a route error.
    /// Folds for both the fresh dispatch and the [`redispatch_held`](Self::redispatch_held) retry.
    fn route_and_fold(
        &self,
        messages: Vec<(i32, ShuffleMessage)>,
        full: &mut HashMap<i32, Vec<ShuffleMessage>>,
    ) {
        let mut dispatched = 0u64;
        let mut route_errors = 0u64;
        for (partition, outcome) in self.router.try_route_batch(messages) {
            match outcome {
                SendOutcome::Sent { max_offset, count } => {
                    // Ceiling raised after `try_send`, so a worker could mark this batch processed first
                    // and trip a spurious `CappedAheadOfDispatch`. Harmless (marks are monotonic and
                    // commits derive from `processed`, so the next dispatch corrects it) and near
                    // unreachable (per-event work is ~hundreds of reads; this mark is microseconds off).
                    // Marking before the send would wrongly raise the ceiling for `Full`/dropped
                    // sub-batches. `None` carries no offset, so it leaves the ceiling untouched.
                    if let Some(max_offset) = max_offset {
                        self.tracker.mark_dispatched(partition, max_offset + 1);
                    }
                    dispatched += count as u64;
                }
                SendOutcome::Full(returned) => {
                    full.entry(partition).or_default().extend(returned);
                }
                SendOutcome::NoWorker(returned) | SendOutcome::ChannelClosed(returned) => {
                    full.entry(partition).or_default().extend(returned);
                    route_errors += 1;
                }
            }
        }
        if dispatched > 0 {
            counter!(COHORT_STREAM_EVENTS_DISPATCHED).increment(dispatched);
        }
        if route_errors > 0 {
            counter!(COHORT_STREAM_ROUTE_ERRORS).increment(route_errors);
        }
    }

    /// Snapshot of the partitions currently owned, for the backpressure prune/reconcile pass.
    pub fn owned_set(&self) -> HashSet<i32> {
        self.owned.iter().map(|entry| *entry.key()).collect()
    }

    /// Route a consumed `person_merge_events` batch to per-partition workers, ceiling on the merge
    /// tracker. Spawns workers (unlike sweep) because merges are durable external input.
    pub async fn dispatch_merges(&self, batch: Vec<ConsumedMerge>) {
        if batch.is_empty() {
            return;
        }
        let items = batch
            .into_iter()
            .map(|consumed| {
                (
                    consumed.partition,
                    consumed.offset,
                    ShuffleMessage::Merge {
                        event: consumed.event,
                        offset: consumed.offset,
                    },
                )
            })
            .collect();
        let stats = self
            .dispatch_to_workers(items, &self.merge.merge_tracker)
            .await;
        counter!(COHORT_STREAM_MERGES_SKIPPED_NOT_OWNED).increment(stats.not_owned_skipped);
    }

    /// Route a consumed `cohort_merge_state_transfer` batch to per-partition workers, ceiling on the
    /// transfer tracker. Spawns workers because a dropped transfer is unrecoverable.
    pub async fn dispatch_transfers(&self, batch: Vec<ConsumedTransfer>) {
        if batch.is_empty() {
            return;
        }
        let items = batch
            .into_iter()
            .map(|consumed| {
                (
                    consumed.partition,
                    consumed.offset,
                    ShuffleMessage::Transfer {
                        transfer: Box::new(consumed.transfer),
                        offset: consumed.offset,
                    },
                )
            })
            .collect();
        let stats = self
            .dispatch_to_workers(items, &self.merge.transfer_tracker)
            .await;
        counter!(COHORT_STREAM_TRANSFERS_SKIPPED_NOT_OWNED).increment(stats.not_owned_skipped);
    }

    /// Route a consumed `cohort_cascade_events` batch to per-partition workers, ceiling on the
    /// cascade tracker. Spawns workers because a dropped cascade silently fails to propagate a flip.
    pub async fn dispatch_cascade(&self, batch: Vec<ConsumedCascade>) {
        if batch.is_empty() {
            return;
        }
        let items = batch
            .into_iter()
            .map(|consumed| {
                (
                    consumed.partition,
                    consumed.offset,
                    ShuffleMessage::Cascade {
                        message: Box::new(consumed.message),
                        offset: consumed.offset,
                    },
                )
            })
            .collect();
        let stats = self
            .dispatch_to_workers(items, &self.merge.cascade_tracker)
            .await;
        counter!(COHORT_STREAM_CASCADES_SKIPPED_NOT_OWNED).increment(stats.not_owned_skipped);
    }

    /// Non-blocking seed dispatch onto the workers' seed lanes, raising the **seed** tracker's
    /// ceiling only for what lands. A sub-batch may land as a prefix, so the ceiling comes from the
    /// router's report of the highest offset that reached the lane, never from the batch maximum.
    /// Returns the un-dispatched seeds for the caller to hold and pause.
    pub fn dispatch_seeds(&self, batch: Vec<ConsumedSeed>) -> HashMap<i32, Vec<ConsumedSeed>> {
        if self.draining.load(Ordering::SeqCst) {
            if !batch.is_empty() {
                counter!(COHORT_STREAM_SEEDS_SKIPPED_NOT_OWNED).increment(batch.len() as u64);
                debug!(
                    dropped = batch.len(),
                    "seed dispatch after shutdown began; dropping batch for replay"
                );
            }
            return HashMap::new();
        }

        let mut owned_seeds: Vec<ConsumedSeed> = Vec::with_capacity(batch.len());
        let mut not_owned = 0u64;
        for consumed in batch {
            if !self.owned.contains_key(&consumed.partition) {
                not_owned += 1;
                continue;
            }
            self.ensure_worker(consumed.partition);
            owned_seeds.push(consumed);
        }
        counter!(COHORT_STREAM_SEEDS_SKIPPED_NOT_OWNED).increment(not_owned);

        let mut full: HashMap<i32, Vec<ConsumedSeed>> = HashMap::new();
        let mut route_errors = 0u64;
        for (partition, outcome) in self.router.try_route_seeds(owned_seeds) {
            let landed_max = match outcome {
                SeedSendOutcome::Sent { max_offset } => Some(max_offset),
                SeedSendOutcome::Refused {
                    landed_max,
                    reason,
                    rest,
                } => {
                    if reason != SeedRefusal::Full {
                        route_errors += 1;
                    }
                    full.entry(partition).or_default().extend(rest);
                    landed_max
                }
            };
            if let Some(max_offset) = landed_max {
                self.merge
                    .seed_tracker
                    .mark_dispatched(partition, max_offset + 1);
            }
        }
        if route_errors > 0 {
            counter!(COHORT_STREAM_ROUTE_ERRORS).increment(route_errors);
        }
        full
    }

    async fn dispatch_to_workers(
        &self,
        items: Vec<(i32, i64, ShuffleMessage)>,
        tracker: &OffsetTracker,
    ) -> DispatchStats {
        let mut stats = DispatchStats::default();
        if self.draining.load(Ordering::SeqCst) {
            stats.not_owned_skipped = items.len() as u64;
            debug!(
                dropped = items.len(),
                "dispatch after shutdown began; dropping batch for replay"
            );
            return stats;
        }

        let mut messages: Vec<(i32, ShuffleMessage)> = Vec::with_capacity(items.len());
        for (partition, offset, message) in items {
            if !self.owned.contains_key(&partition) {
                stats.not_owned_skipped += 1;
                continue;
            }
            self.ensure_worker(partition);
            tracker.mark_dispatched(partition, offset + 1); // next-to-consume convention
            messages.push((partition, message));
        }
        stats.dispatched = messages.len() as u64;

        let errors = self.router.route_batch(messages).await;
        if !errors.is_empty() {
            counter!(COHORT_STREAM_ROUTE_ERRORS).increment(errors.len() as u64);
        }
        stats
    }

    /// Spawn a worker the first time an owned partition delivers; no-op if the worker exists or the
    /// partition is unowned. The ownership check and insert are atomic under the DashMap shard guard.
    fn ensure_worker(&self, partition: i32) {
        match self.workers.entry(partition) {
            Entry::Occupied(_) => {}
            Entry::Vacant(slot) => {
                // No `.await` in this arm — the DashMap shard guard is held.
                let Some(assigned_at) = self.owned.get(&partition).map(|entry| *entry) else {
                    return;
                };
                if self.draining.load(Ordering::SeqCst) {
                    return;
                }
                match self.router.add_partition(partition) {
                    Some(inbox) => {
                        let restore = if self.durable_restore_enabled() {
                            EvictionRestore::Rebuild(self.readiness.rebuild_started())
                        } else {
                            EvictionRestore::Skip
                        };
                        let worker = Stage1Worker::spawn_with_gating(
                            partition as u16,
                            inbox,
                            self.handle.clone(),
                            self.catalog.clone(),
                            self.sink.clone(),
                            self.tracker.clone(),
                            self.merge.clone(),
                            restore,
                            self.event_name_gating(),
                            assigned_at,
                        );
                        slot.insert(worker);
                        counter!(COHORT_STREAM_WORKERS_SPAWNED).increment(1);
                        info!(
                            partition,
                            "spawned stage 1 worker for newly-delivered partition"
                        );
                    }
                    None if self.router.is_closed() => {
                        debug!(
                            partition,
                            "router closed by shutdown; skipping worker spawn"
                        )
                    }
                    None => warn!(
                        partition,
                        "router holds a live channel but no worker is registered; skipping spawn",
                    ),
                }
            }
        }
    }

    pub fn owns(&self, partition: i32) -> bool {
        self.owned.contains_key(&partition)
    }

    /// End boot and open the readiness gate. Every owned partition gets its worker first, so the
    /// gate already counts their eviction rebuilds when it opens, and an idle partition rebuilds
    /// and sweeps from boot instead of from its first message. Returns the owned partition count.
    pub(crate) fn finish_boot(&self) -> usize {
        let owned = self.owned_set();
        for &partition in &owned {
            self.ensure_worker(partition);
        }
        self.readiness.mark_live();
        owned.len()
    }

    /// Snapshot of the partitions currently owned by this consumer.
    pub fn owned_partitions(&self) -> Vec<i32> {
        self.owned.iter().map(|entry| *entry.key()).collect()
    }

    pub(crate) fn merge_deps(&self) -> &MergeWorkerDeps {
        &self.merge
    }

    /// Route a sweep tick to each owned partition that has a live worker. Never spawns: a worker-less
    /// owned partition has no in-memory state to evict, so it is skipped rather than producing a
    /// `no_worker` drop.
    ///
    /// Routes nothing before the first catalog load. Every team is absent from the unloaded catalog,
    /// and a sweep batch drops the keys it claims for an absent team without rescheduling them, so a
    /// sweep then would lose every deadline the boot rebuild restored.
    pub async fn route_sweep(&self, due_before_ms: i64) {
        if !self.catalog.is_loaded() {
            return;
        }
        self.route_to_owned(|| ShuffleMessage::Sweep { due_before_ms })
            .await;
    }

    /// Route a redrive tick to each owned partition that has a live worker, without spawning; each
    /// worker re-produces any `cf_pending_transfers` entries stranded by inline-retry exhaustion.
    pub async fn route_redrive(&self) {
        self.route_to_owned(|| ShuffleMessage::RedrivePendingTransfers)
            .await;
    }

    /// Route a merge-GC tick to each owned partition that has a live worker, without spawning. Cutoffs
    /// are computed by the sweeper and passed verbatim, keeping workers clock-free.
    pub async fn route_merge_gc(&self, marker_cutoff_ms: i64, tombstone_cutoff_ms: i64) {
        self.route_to_owned(|| ShuffleMessage::MergeCfGc {
            marker_cutoff_ms,
            tombstone_cutoff_ms,
        })
        .await;
    }

    /// Route one bounded reconcile-drain tick to each owned partition with a live worker.
    pub async fn route_reconcile_drain(&self) {
        self.route_to_owned(|| ShuffleMessage::ReconcileDrain).await;
    }

    /// Owned partitions that currently have a live worker channel — the only partitions a maintenance
    /// tick can reach. No owned partition has a worker before boot ends, so a tick to one then is a
    /// guaranteed `no_worker` no-op (no in-memory state to evict, redrive, or GC); skipping it keeps
    /// the fan-out silent instead of dropping one message per worker-less partition.
    fn tickable_partitions(&self) -> Vec<i32> {
        self.owned_partitions()
            .into_iter()
            .filter(|partition| self.router.has_partition(*partition))
            .collect()
    }

    async fn route_to_owned(&self, make_message: impl Fn() -> ShuffleMessage) {
        if self.draining.load(Ordering::SeqCst) {
            return;
        }
        let messages: Vec<(i32, ShuffleMessage)> = self
            .tickable_partitions()
            .into_iter()
            .map(|partition| (partition, make_message()))
            .collect();
        if messages.is_empty() {
            return;
        }
        let errors = self.router.route_batch(messages).await;
        if !errors.is_empty() {
            counter!(COHORT_STREAM_ROUTE_ERRORS).increment(errors.len() as u64);
        }
    }

    pub fn assign_partition(&self, partition: i32) {
        // Forget before ownership flips: every tenure starts fail-closed for the seed fence, and
        // a concurrent idle-probe advance can't slip in between.
        self.merge.live_watermarks.forget_partition(partition);
        self.owned.insert(partition, CoverageStartMs::now());
        counter!(PARTITIONS_ASSIGNED_TOTAL).increment(1);
    }

    /// Delete the on-disk slices of every partition `ownership` does not own, and return how many
    /// held any key. The store opens only under the ownership it was written under, or adopts it,
    /// so at one pod this deletes nothing.
    pub(crate) async fn reclaim_unowned_slices(&self, ownership: &PartitionOwnership) -> u64 {
        let mut wiped = 0u64;
        for partition in (0..ownership.partition_count()).filter(|&p| !ownership.owns(p)) {
            match self.handle.delete_partition(partition).await {
                Ok(held_keys) => wiped += u64::from(held_keys),
                Err(err) => warn!(
                    partition,
                    error = %err, "failed to reclaim unowned-partition state at boot"
                ),
            }
        }
        let kept = ownership.partitions().count() as u64;
        counter!(DURABLE_RESTORE_PARTITIONS_KEPT_TOTAL).increment(kept);
        counter!(DURABLE_RESTORE_PARTITIONS_WIPED_STALE_TOTAL).increment(wiped);
        info!(
            kept,
            wiped,
            partition_count = ownership.partition_count(),
            "durable restore: boot staleness sweep complete"
        );
        wiped
    }

    /// Re-produce every staged `cf_pending_transfers` entry for the owned partitions at boot, before
    /// any worker spawns. This paginates the whole outbox directly and re-produces every entry
    /// regardless of backlog size.
    ///
    /// Produce + clear only — no `mark_processed`: a fresh tenure starts at `dispatched_offset == 0`
    /// so marking would clamp to 0 and trip `CappedAheadOfDispatch`. A per-entry produce failure
    /// leaves that entry for the periodic redrive. Must run after `reclaim_unowned_slices`.
    pub async fn eager_redrive_pending_transfers_on_boot(&self, assignment: &HashSet<i32>) {
        for &partition in assignment {
            let Some(store_partition) = partition_to_store_id(partition) else {
                warn!(
                    partition,
                    "eager boot redrive: partition out of u16 range; skipping"
                );
                continue;
            };
            let recovered = self
                .eager_redrive_partition(partition, store_partition, EAGER_BOOT_REDRIVE_PAGE_SIZE)
                .await;
            if recovered > 0 {
                counter!(DURABLE_RESTORE_PENDING_TRANSFERS_RECOVERED_PARTITIONS_TOTAL).increment(1);
                info!(
                    partition,
                    recovered, "eager boot redrive: re-produced stranded pending transfers",
                );
            }
        }
    }

    /// Paginate one partition's `cf_pending_transfers` outbox, re-producing and clearing each entry;
    /// returns the count re-produced. `page_size` is a parameter so tests can force multi-page
    /// pagination without seeding a full [`EAGER_BOOT_REDRIVE_PAGE_SIZE`] page.
    async fn eager_redrive_partition(
        &self,
        partition: i32,
        store_partition: u16,
        page_size: usize,
    ) -> usize {
        let mut cursor: Option<Vec<u8>> = None;
        let mut recovered = 0usize;
        loop {
            let page = match self
                .handle
                .scan_pending_transfers(store_partition, cursor.clone(), page_size)
                .await
            {
                Ok(page) => page,
                Err(error) => {
                    warn!(
                        partition,
                        error = %error,
                        "eager boot redrive: pending-transfer scan failed; the periodic redrive retries it",
                    );
                    break;
                }
            };
            if page.is_empty() {
                break;
            }
            let page_len = page.len();
            // Advance past the whole page before processing it: an entry left in place (decode or
            // produce failure) is never re-scanned this boot, so the loop always terminates and the
            // stranded entry defers to the periodic redrive.
            let next_cursor = page.last().map(|(key, _)| key.encode().to_vec());

            for (key, bytes) in page {
                let pending = match PendingTransfer::decode(&bytes) {
                    Ok(pending) => pending,
                    Err(error) => {
                        warn!(
                            partition,
                            team_id = key.team_id,
                            old_person = %key.old_person,
                            error = %error,
                            "eager boot redrive: undecodable pending transfer; leaving it in place",
                        );
                        continue;
                    }
                };
                let team_id = pending.transfer.team_id;
                let old_person = pending.transfer.old_person_uuid;
                let acks = self
                    .merge
                    .transfer_sink
                    .produce(vec![pending.transfer])
                    .await;
                if !acks.iter().all(Result::is_ok) {
                    warn!(
                        partition,
                        team_id,
                        old_person = %old_person,
                        "eager boot redrive: transfer produce failed; leaving the entry for the periodic redrive",
                    );
                    continue;
                }
                if let Err(error) = self.handle.clear_pending_transfer(&key).await {
                    warn!(
                        partition,
                        team_id,
                        old_person = %old_person,
                        error = %error,
                        "eager boot redrive: outbox clear failed after an acked produce; the periodic redrive re-produces it idempotently",
                    );
                }
                recovered += 1;
            }

            cursor = next_cursor;
            if page_len < page_size {
                break;
            }
        }
        recovered
    }

    #[cfg(test)]
    fn tracker(&self) -> &OffsetTracker {
        self.tracker.as_ref()
    }

    /// The events-topic tracker, for the seed consumer's idle probe.
    pub(crate) fn events_tracker(&self) -> &Arc<OffsetTracker> {
        &self.tracker
    }

    fn owned_committable_offsets(&self) -> HashMap<i32, i64> {
        self.tracker
            .committable_offsets()
            .into_iter()
            .filter(|(partition, _)| self.owned.contains_key(partition))
            .collect()
    }

    /// Drain all workers, publish the ownership they drained under for the followers' final
    /// commits, and return the tracker. No consumer polls during the drain, so no rebalance callback
    /// can change `owned` while it runs.
    async fn shutdown(&self) -> Arc<OffsetTracker> {
        self.draining.store(true, Ordering::SeqCst);
        self.router.clear();
        let started = Instant::now();
        let partitions: Vec<i32> = self.workers.iter().map(|entry| *entry.key()).collect();
        let mut workers = 0usize;
        for partition in partitions {
            if let Some((_, worker)) = self.workers.remove(&partition) {
                workers += 1;
                if let Err(err) = worker.join().await {
                    warn!(partition, error = %err, "stage 1 worker panicked during shutdown drain");
                }
            }
        }
        info!(
            workers,
            elapsed_ms = started.elapsed().as_millis() as u64,
            "worker drain complete"
        );
        if self
            .drained
            .set(DrainedOwnership(self.owned_set()))
            .is_err()
        {
            debug!("worker drain already published; keeping the first");
        }
        self.tracker.clone()
    }

    /// Wait up to `budget` for [`Self::shutdown`] to finish draining the workers.
    pub(crate) async fn wait_for_worker_drain(&self, budget: Duration) -> DrainWait {
        match tokio::time::timeout(budget, self.drained.wait()).await {
            Ok(drained) => DrainWait::Drained(drained.clone()),
            Err(_) => DrainWait::TimedOut,
        }
    }
}

#[derive(Debug, Default)]
struct DispatchStats {
    dispatched: u64,
    not_owned_skipped: u64,
}

fn partition_to_store_id(partition: i32) -> Option<u16> {
    u16::try_from(partition).ok()
}

/// The `cohort_stream_events` consumer: consume, route, commit.
///
/// Uses a raw `StreamConsumer` with manual commit because the per-partition `OffsetTracker`
/// commits a `TopicPartitionList` that the `common-kafka` wrapper can't express.
pub struct CohortStreamEventsConsumer {
    /// `Arc` so the commit task and consume loop share one consumer; rdkafka supports concurrent
    /// `recv` and `commit`.
    consumer: Arc<StreamConsumer<CohortConsumerContext>>,
    topic: String,
    dispatcher: Arc<EventDispatcher>,
    handle: Handle,
    /// Pauses/resumes events partitions to express downstream backpressure as lag.
    pauser: Arc<dyn PartitionPauser>,
    recv_batch_size: usize,
    recv_batch_timeout: Duration,
    offset_commit_interval: Duration,
    claimed: ClaimedSplit,
    /// A checkpoint restore boot must finish. Boot rewinds every owned partition the restore keeps
    /// to its restored position, commits, and finishes it before anything folds.
    restore: Option<PendingRestore>,
}

impl CohortStreamEventsConsumer {
    /// Takes the partitions `claimed` owns for the life of the process: the consumer is assigned
    /// them at its group's committed offsets, and the dispatcher owns them.
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        consumer: Arc<StreamConsumer<CohortConsumerContext>>,
        topic: String,
        dispatcher: Arc<EventDispatcher>,
        handle: Handle,
        recv_batch_size: usize,
        recv_batch_timeout: Duration,
        offset_commit_interval: Duration,
        claimed: &ClaimedSplit,
        restore: Option<PendingRestore>,
    ) -> KafkaResult<Self> {
        let ownership = claimed.ownership();
        consumer.assign(&ownership.assignment(&topic)?)?;
        for partition in ownership.partitions() {
            dispatcher.assign_partition(i32::from(partition));
        }
        let pods: Arc<str> = Arc::from(ownership.pods().to_string());
        let ordinal: Arc<str> = Arc::from(ownership.ordinal().to_string());
        gauge!(PARTITION_OWNERSHIP_INFO, "pods" => pods, "ordinal" => ordinal).set(1.0);
        gauge!(OWNED_PARTITIONS).set(ownership.partitions().count() as f64);
        info!(%ownership, topic = %topic, "assigned the owned partitions");

        let pauser: Arc<dyn PartitionPauser> =
            Arc::new(ConsumerPauser::new(consumer.clone(), topic.clone()));
        Ok(Self {
            consumer,
            topic,
            dispatcher,
            handle,
            pauser,
            recv_batch_size,
            recv_batch_timeout,
            offset_commit_interval,
            claimed: claimed.clone(),
            restore,
        })
    }

    /// Run until the lifecycle handle signals shutdown.
    ///
    /// Offset commit runs on its own interval task so a CPU-saturated consume loop or worker backlog
    /// can't block offset advancement; it shares the shutdown signal and is awaited before the final
    /// synchronous commit. Liveness is reported after every poll without a transport error, except
    /// while boot is still rewinding.
    pub async fn process(mut self) {
        let _guard = self.handle.process_scope();
        info!(topic = %self.topic, "cohort_stream_events consume loop starting");
        // Lifecycle ignores a component until its first report, so this starts the liveness clock
        // before boot recovery: a recovery or a rewind that never finishes then restarts the process.
        self.handle.report_healthy();

        let commit_task = tokio::spawn(run_commit_loop(
            self.consumer.clone(),
            self.dispatcher.clone(),
            self.claimed.stamp(),
            self.topic.clone(),
            self.offset_commit_interval,
            self.handle.clone(),
        ));

        // Apply pause/resume off the consume loop: librdkafka's are synchronous, un-timed FFI, so a
        // wedged client must never delay the heartbeat. The loop hands over the paused *target* (the
        // held-partition set); the pauser task owns re-asserting it.
        let (pause_tx, pause_rx) = tokio::sync::mpsc::unbounded_channel::<HashSet<i32>>();
        let pauser_task = tokio::spawn(run_pauser_loop(self.pauser.clone(), pause_rx));

        let boot_started = Instant::now();
        // `validate_startup` refuses a checkpoint restore without durable restore, so a restore
        // always comes with it. Without durable restore the store started empty: nothing to recover.
        let mut phase = match (
            self.dispatcher.durable_restore_enabled(),
            self.restore.take(),
        ) {
            (false, None) => self.go_live(boot_started, 0),
            (true, None) => {
                self.recover().await;
                self.go_live(boot_started, 0)
            }
            (_, Some(restore)) => {
                self.recover().await;
                self.wait_for_capture(&restore).await;
                let mut resume = ResumePoints::default();
                let topic = InputTopic::new(self.topic.as_str());
                resume.rewind_to(&restore.plan().positions(&topic));
                BootPhase::Rewinding {
                    resume,
                    restore: Box::new(restore),
                }
            }
        };
        // Untouched until boot recovery completes, so nothing pauses during boot.
        let mut backpressure = Backpressure::new();
        // Last paused target sent to the pauser task: lets us send only while backpressure is active
        // (plus the cycle it clears) and stay silent otherwise.
        let mut prev_paused_target: HashSet<i32> = HashSet::new();

        loop {
            tokio::select! {
                biased;
                _ = self.handle.shutdown_recv() => {
                    info!("shutdown signal received, stopping consume loop");
                    break;
                }
                outcome = self.consume_batch() => {
                    let transport_error = outcome.transport_error;
                    phase = match phase {
                        BootPhase::Live => {
                            self.dispatch_with_backpressure(
                                outcome,
                                &mut backpressure,
                                &mut prev_paused_target,
                                &pause_tx,
                            );
                            BootPhase::Live
                        }
                        BootPhase::Rewinding { mut resume, restore } => {
                            resume.hold_back(&outcome.events);
                            self.rewind(resume, restore, boot_started)
                        }
                        BootPhase::Halted => BootPhase::Halted,
                    };
                    if transport_error {
                        tokio::time::sleep(RECV_ERROR_BACKOFF).await;
                    } else if !matches!(phase, BootPhase::Rewinding { .. }) {
                        self.handle.report_healthy();
                    }
                }
            }
        }

        // Await the commit task before the final synchronous commit so the two don't race.
        if let Err(err) = commit_task.await {
            warn!(error = %err, "offset-commit task did not exit cleanly");
        }
        // Drop the sender so the pauser task drains and exits.
        drop(pause_tx);
        if let Err(err) = pauser_task.await {
            warn!(error = %err, "pauser task did not exit cleanly");
        }

        self.dispatcher.shutdown().await;
        let offsets = self.dispatcher.owned_committable_offsets();
        fsync_then_commit(
            self.dispatcher.handle(),
            &self.consumer,
            &self.claimed.stamp(),
            offsets,
            &self.topic,
            CommitMode::Sync,
        )
        .await;
        info!(topic = %self.topic, "cohort_stream_events consume loop stopped");
    }

    /// Boot recovery of a reopened or restored store, before the first poll: delete the slices of
    /// the partitions this pod does not own, and re-produce the merge outbox of those it owns.
    async fn recover(&self) {
        self.dispatcher
            .reclaim_unowned_slices(&self.claimed.ownership())
            .await;
        self.dispatcher
            .eager_redrive_pending_transfers_on_boot(&self.dispatcher.owned_set())
            .await;
    }

    /// Nothing polls or folds while this host's clock reads before the checkpoint's capture, so no
    /// fold emits a version below the outputs that made the restored state. The source pod can emit
    /// more after the capture, and only the wall clocks of the two hosts order those outputs before
    /// this pod's. A shutdown ends the wait.
    async fn wait_for_capture(&self, restore: &PendingRestore) {
        let captured_at = restore.captured_at();
        while let Ok(wait) = (captured_at - Utc::now()).to_std() {
            warn!(
                %captured_at,
                wait_secs = wait.as_secs(),
                "this host's clock reads before the restored checkpoint's capture; holding off \
                 dispatch until it passes",
            );
            tokio::select! {
                biased;
                _ = self.handle.shutdown_recv() => return,
                _ = tokio::time::sleep(wait) => {}
            }
        }
    }

    /// Seek every owned partition back to its resume point, finish the restore, then go live. Any
    /// failure keeps boot in `Rewinding`, so the next poll retries and nothing folds from the
    /// broker offset. `Rewinding` reports no liveness, so a persistent failure restarts the process.
    ///
    /// The restore commits the seek list before it finishes. Finishing deletes the restore marker,
    /// so without the commit a crash right after would reopen the restored store at the broker's
    /// old, higher offsets and skip the events between.
    fn rewind(
        &self,
        resume: ResumePoints,
        restore: Box<PendingRestore>,
        boot_started: Instant,
    ) -> BootPhase {
        let owned = self.dispatcher.owned_set();
        let seek_list = match resume.seek_list(&self.claimed.stamp(), &self.topic, &owned) {
            Ok(seek_list) => seek_list,
            Err(err) => {
                warn!(
                    topic = %self.topic,
                    error = %err,
                    "boot seek list rejected an offset; holding off dispatch and retrying",
                );
                return BootPhase::Rewinding { resume, restore };
            }
        };
        let sought = seek_list
            .as_ref()
            .map_or(0, |seek_list| seek_list.as_list().count());

        if let Some(seek_list) = &seek_list {
            let failed: Vec<i32> = match self
                .consumer
                .seek_partitions(seek_list.as_list().clone(), BOOT_SEEK_TIMEOUT)
            {
                Ok(result) => result
                    .elements_for_topic(&self.topic)
                    .iter()
                    .filter(|elem| elem.error().is_err())
                    .map(|elem| elem.partition())
                    .collect(),
                Err(err) => {
                    warn!(
                        topic = %self.topic,
                        partitions = sought,
                        error = %err,
                        "boot seek failed; holding off dispatch and retrying (no progress until it succeeds)",
                    );
                    return BootPhase::Rewinding { resume, restore };
                }
            };
            if !failed.is_empty() {
                warn!(
                    topic = %self.topic,
                    failed_partitions = ?failed,
                    "boot seek failed for some partitions; holding off dispatch and retrying (no progress until it succeeds)",
                );
                return BootPhase::Rewinding { resume, restore };
            }
        }

        let low_watermarks = |partitions: &[(&InputTopic, u16)]| {
            list_offsets(self.consumer.as_ref(), partitions, Offset::Beginning)
        };
        let commit_events = || match &seek_list {
            Some(seek_list) => seek_list.commit(self.consumer.as_ref(), CommitMode::Sync),
            None => Ok(()),
        };
        // A sync commit waits until the group coordinator answers. `block_in_place` hands this
        // worker's other tasks to another thread meanwhile, so a runtime with one worker still
        // serves the health server.
        match tokio::task::block_in_place(|| restore.finish(&owned, low_watermarks, commit_events))
        {
            Ok(()) => {
                gauge!(CHECKPOINT_RESTORE_BLOCKED).set(0.0);
                info!(
                    topic = %self.topic,
                    partitions = sought,
                    "checkpoint restore finished: the events group committed the restored positions",
                );
            }
            Err(FinishError::Expired(expired)) => {
                // The marker stays, so the next boot's window check resets these slices before
                // anything reads them.
                error!(
                    ?expired,
                    "the broker expired restored positions during boot; stopping so the next boot resets those slices",
                );
                self.handle
                    .signal_failure("checkpoint restore positions expired during boot");
                return BootPhase::Halted;
            }
            Err(err) => {
                gauge!(CHECKPOINT_RESTORE_BLOCKED).set(1.0);
                warn!(
                    topic = %self.topic,
                    error = %format!("{:#}", anyhow::Error::new(err)),
                    "checkpoint restore could not finish; holding off dispatch and retrying",
                );
                return BootPhase::Rewinding { resume, restore };
            }
        }
        self.go_live(boot_started, sought)
    }

    fn go_live(&self, boot_started: Instant, sought: usize) -> BootPhase {
        let partitions = self.dispatcher.finish_boot();
        info!(
            elapsed_ms = boot_started.elapsed().as_millis() as u64,
            partitions, sought, "events consumer boot complete",
        );
        BootPhase::Live
    }

    /// One steady-state cycle: prune revoked holdover, retry-flush held partitions, dispatch the polled
    /// batch, reconcile the paused set and hand its deltas to the pauser task. Every step is
    /// non-blocking, so the loop's heartbeat fires each iteration regardless of downstream drain.
    fn dispatch_with_backpressure(
        &self,
        outcome: ConsumeOutcome,
        backpressure: &mut Backpressure,
        prev_paused_target: &mut HashSet<i32>,
        pause_tx: &tokio::sync::mpsc::UnboundedSender<HashSet<i32>>,
    ) {
        histogram!(COHORT_STREAM_CONSUME_BATCH_SIZE).record(outcome.events.len() as f64);
        if !outcome.events.is_empty() {
            counter!(COHORT_STREAM_EVENTS_CONSUMED).increment(outcome.events.len() as u64);
        }
        if outcome.deserialize_errors > 0 {
            counter!(COHORT_STREAM_DESERIALIZE_ERRORS).increment(outcome.deserialize_errors);
        }
        if outcome.empty_payloads > 0 {
            counter!(COHORT_STREAM_EMPTY_PAYLOAD).increment(outcome.empty_payloads);
        }

        let owned = self.dispatcher.owned_set();
        backpressure.prune_revoked(&owned);
        // Retry-flush held partitions before dispatching new events, so a newer offset never leapfrogs
        // an older held one on the same partition.
        let still_full = self.dispatcher.redispatch_held(backpressure.take_held());
        backpressure.absorb(still_full);
        let full = self
            .dispatcher
            .dispatch_events_nonblocking(outcome.events, &backpressure.held_partitions());
        backpressure.absorb(full);

        // Hand the pauser task the paused target (the still-held partitions); it re-asserts pause
        // idempotently, so no librdkafka call can delay the heartbeat below. Send every cycle the
        // target is non-empty (re-asserting heals a swallowed pause error), plus the one cycle it
        // clears (to resume); stay silent otherwise.
        let target = backpressure.held_partitions();
        if !target.is_empty() || !prev_paused_target.is_empty() {
            if pause_tx.send(target.clone()).is_err() {
                // The receiver is gone only at shutdown, where an unapplied target is moot.
                debug!("pauser task has exited; skipping a pause/resume update");
            }
            *prev_paused_target = target;
        }
        gauge!(PARTITIONS_PAUSED).set(backpressure.held_partition_count() as f64);
        gauge!(PENDING_HELD_EVENTS).set(backpressure.held_message_count() as f64);
    }

    #[hotpath::measure(impl_type = "CohortStreamEventsConsumer")]
    async fn consume_batch(&self) -> ConsumeOutcome {
        let mut outcome = ConsumeOutcome {
            events: Vec::with_capacity(self.recv_batch_size),
            deserialize_errors: 0,
            empty_payloads: 0,
            transport_error: false,
        };

        tokio::select! {
            _ = tokio::time::sleep(self.recv_batch_timeout) => {}
            _ = async {
                while outcome.events.len() < self.recv_batch_size {
                    match self.consumer.recv().await {
                        Ok(message) => {
                            let partition = message.partition();
                            let offset = message.offset();
                            match message.payload() {
                                None => {
                                    outcome.empty_payloads += 1;
                                    debug!(
                                        partition, offset,
                                        "skipping cohort_stream_events message with empty payload",
                                    );
                                }
                                Some(payload) => match serde_json::from_slice::<CohortStreamEvent>(payload) {
                                    Ok(event) => outcome.events.push(ConsumedEvent {
                                        event,
                                        partition,
                                        offset,
                                        broker_ts_ms: message.timestamp().to_millis(),
                                    }),
                                    Err(err) => {
                                        outcome.deserialize_errors += 1;
                                        debug!(
                                            partition, offset, error = %err,
                                            "skipping undeserializable cohort_stream_events message",
                                        );
                                    }
                                },
                            }
                        }
                        Err(err) => {
                            outcome.transport_error = true;
                            counter!(COHORT_STREAM_KAFKA_RECV_ERRORS).increment(1);
                            warn!(error = %err, "kafka recv error while consuming cohort_stream_events");
                            break;
                        }
                    }
                }
            } => {}
        }

        outcome
    }
}

struct ConsumeOutcome {
    events: Vec<ConsumedEvent>,
    deserialize_errors: u64,
    empty_payloads: u64,
    transport_error: bool,
}

pub(crate) fn commit_offsets<C: ConsumerContext>(
    consumer: &StreamConsumer<C>,
    stamp: &CommitStamp,
    offsets: HashMap<i32, i64>,
    topic: &str,
    mode: CommitMode,
) {
    if offsets.is_empty() {
        return;
    }
    match stamp
        .list(topic, offsets)
        .and_then(|list| list.commit(consumer, mode))
    {
        Ok(()) => counter!(COHORT_STREAM_OFFSET_COMMITS).increment(1),
        Err(err) => {
            counter!(COHORT_STREAM_OFFSET_COMMIT_ERRORS).increment(1);
            warn!(topic, error = %err, "failed to commit consumer offsets");
        }
    }
}

/// fsync the store's WAL before committing offsets, upholding `committed <= durable`. Unconditional
/// so reopen-live is safe whenever the durability gate is flipped. A fsync error skips the commit.
///
/// The caller captures `offsets` before this runs, so the fsync makes durable exactly what they
/// already reflect. It runs on the write lane with no permit so the commit cadence never queues
/// behind reads.
pub(crate) async fn fsync_then_commit<C: ConsumerContext>(
    handle: &StoreHandle,
    consumer: &StreamConsumer<C>,
    stamp: &CommitStamp,
    offsets: HashMap<i32, i64>,
    topic: &str,
    mode: CommitMode,
) {
    if offsets.is_empty() {
        return;
    }
    if handle.flush_wal_sync().await.is_err() {
        return; // store counted the error; skip commit so `committed` never outruns `durable`
    }
    commit_offsets(consumer, stamp, offsets, topic, mode);
}

/// Flushes the store's WAL and commits the owned committable offsets on a fixed interval, independent
/// of the consume loop so a stalled loop can't block offset advancement. Exits on shutdown.
async fn run_commit_loop(
    consumer: Arc<StreamConsumer<CohortConsumerContext>>,
    dispatcher: Arc<EventDispatcher>,
    stamp: CommitStamp,
    topic: String,
    interval: Duration,
    handle: Handle,
) {
    let mut ticker = tokio::time::interval(interval);
    ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
    loop {
        tokio::select! {
            biased;
            _ = handle.shutdown_recv() => break,
            _ = ticker.tick() => {
                fsync_then_commit(
                    dispatcher.handle(),
                    &consumer,
                    &stamp,
                    dispatcher.owned_committable_offsets(),
                    &topic,
                    CommitMode::Async,
                )
                .await;
            }
        }
    }
}

/// Applies the paused-partition target off the consume loop, so blocking librdkafka `pause`/`resume`
/// never delays the heartbeat. Re-asserts the full target every update — idempotent in librdkafka — so
/// a swallowed pause error self-heals. `applied` tracks only which partitions to resume (those that
/// left the target). Exits when the sender is dropped.
pub(crate) async fn run_pauser_loop(
    pauser: Arc<dyn PartitionPauser>,
    mut rx: tokio::sync::mpsc::UnboundedReceiver<HashSet<i32>>,
) {
    let mut applied: HashSet<i32> = HashSet::new();
    while let Some(target) = rx.recv().await {
        let to_pause: Vec<i32> = target.iter().copied().collect();
        let to_resume: Vec<i32> = applied.difference(&target).copied().collect();
        // Run the blocking FFI off the worker threads, awaited serially so updates keep their
        // order.
        let ffi_pauser = pauser.clone();
        let applied_ffi = tokio::task::spawn_blocking(move || {
            ffi_pauser.pause(&to_pause);
            ffi_pauser.resume(&to_resume);
        })
        .await;
        match applied_ffi {
            Ok(()) => applied = target,
            Err(err) => {
                // The task may have died anywhere between pause and resume, so fold the target
                // in rather than replace: every possibly-paused partition stays eligible for a
                // future resume, and resuming a never-paused partition is a librdkafka no-op.
                applied.extend(target);
                warn!(error = %err, "pauser task's blocking pause/resume panicked; retrying the delta on the next target update");
            }
        }
    }
}

#[cfg(test)]
// Tests seed and assert against `CohortStore` directly while the dispatchers hold the `StoreHandle`
// facade.
#[allow(clippy::disallowed_methods)]
mod tests {
    use std::collections::HashSet;
    use std::num::{NonZeroU16, NonZeroUsize};

    use super::*;
    use chrono_tz::UTC;
    use common_kafka::kafka_producer::KafkaProduceError;
    use common_types::cohort::TeamAllowlist;
    use futures::FutureExt;
    use serde_json::json;
    use tempfile::TempDir;
    use uuid::Uuid;

    use cohort_core::seed::CoverageStartMs;

    use crate::consumers::readiness::NotReady;
    use crate::filters::{CohortId, FilterCatalog, TeamFiltersBuilder, TeamId};
    use crate::merge::transfer::{
        MergeStateTransfer, PendingTransfer, PersonMergeEvent, Tombstone, TransferLeaf,
        MERGE_EVENT_SCHEMA_VERSION,
    };
    use crate::partitions::offset_tracker::MarkOutcome;
    use crate::partitions::partitioner::{partition_of, COHORT_PARTITION_COUNT};
    use crate::producer::{
        CaptureSink, CaptureStreamEventSink, CaptureTransferSink, CohortMembershipChange,
        MembershipStatus,
    };
    use crate::stage1::state::AppliedOffsets;
    use crate::stage1::{Stage1State, StatefulRecord};
    use crate::store::durability::PodOrdinal;
    use crate::store::{
        Behavioral, BehavioralKey, CohortStore, LeafStateKey, MergeAppliedKey, MergeDrainKey,
        OffloadConfig, OffloadMode, PendingTransferKey, SliceCoverage, SliceCoverageKey,
        SliceCoverages, StoreConfig, TombstoneKey,
    };
    use crate::workers::TransferRetryPolicy;

    const TEAM: i32 = 7;
    const BEHAVIORAL_HASH: [u8; 16] = *b"0123456789abcdef";
    const BASE_TS: &str = "2026-05-26 12:34:56.789000";

    struct BlockingMembershipSink {
        entered: tokio::sync::Notify,
        release: tokio::sync::Notify,
        block_first: AtomicBool,
    }

    impl BlockingMembershipSink {
        fn new() -> Arc<Self> {
            Arc::new(Self {
                entered: tokio::sync::Notify::new(),
                release: tokio::sync::Notify::new(),
                block_first: AtomicBool::new(true),
            })
        }
    }

    #[async_trait::async_trait]
    impl MembershipSink for BlockingMembershipSink {
        async fn produce(
            &self,
            changes: Vec<CohortMembershipChange>,
        ) -> Vec<Result<(), KafkaProduceError>> {
            if self.block_first.swap(false, Ordering::SeqCst) {
                self.entered.notify_one();
                self.release.notified().await;
            }
            changes.into_iter().map(|_| Ok(())).collect()
        }
    }

    /// Wrap a test store in the `All` operating point so the dispatcher runs on the blocking pool.
    fn test_handle(store: &CohortStore) -> StoreHandle {
        StoreHandle::new(
            store.clone(),
            OffloadConfig {
                mode: OffloadMode::All,
                event_read_permits: 16,
                maintenance_permits: 6,
            },
        )
    }

    /// Broker-free [`PartitionPauser`] recording every pause/resume call, to exercise the pauser task
    /// without a Kafka client.
    #[derive(Default)]
    struct RecordingPauser {
        paused: std::sync::Mutex<Vec<Vec<i32>>>,
        resumed: std::sync::Mutex<Vec<Vec<i32>>>,
    }

    impl RecordingPauser {
        /// The non-empty pause calls, in order (the pauser also issues empty no-ops each idle cycle).
        fn nonempty_pauses(&self) -> Vec<Vec<i32>> {
            Self::nonempty(&self.paused)
        }

        fn nonempty_resumes(&self) -> Vec<Vec<i32>> {
            Self::nonempty(&self.resumed)
        }

        fn nonempty(calls: &std::sync::Mutex<Vec<Vec<i32>>>) -> Vec<Vec<i32>> {
            calls
                .lock()
                .unwrap()
                .iter()
                .filter(|call| !call.is_empty())
                .cloned()
                .collect()
        }
    }

    impl PartitionPauser for RecordingPauser {
        fn pause(&self, partitions: &[i32]) {
            self.paused.lock().unwrap().push(partitions.to_vec());
        }

        fn resume(&self, partitions: &[i32]) {
            self.resumed.lock().unwrap().push(partitions.to_vec());
        }
    }

    #[tokio::test]
    async fn pauser_loop_reasserts_the_full_target_each_cycle_and_resumes_on_drop() {
        let pauser = Arc::new(RecordingPauser::default());
        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<HashSet<i32>>();
        let task = tokio::spawn(run_pauser_loop(pauser.clone(), rx));

        // Stable target over two cycles: re-paused each time. That idempotent re-assertion is what
        // heals a swallowed pause error.
        tx.send(HashSet::from([5])).unwrap();
        tx.send(HashSet::from([5])).unwrap();
        // Partition 5 drains and 7 becomes held: 5 leaves the target (resumed), 7 is paused.
        tx.send(HashSet::from([7])).unwrap();
        // Target clears: 7 is resumed.
        tx.send(HashSet::new()).unwrap();
        drop(tx);
        task.await.unwrap();

        assert_eq!(
            pauser.nonempty_pauses(),
            vec![vec![5], vec![5], vec![7]],
            "the full target is re-paused every cycle it is non-empty",
        );
        assert_eq!(
            pauser.nonempty_resumes(),
            vec![vec![5], vec![7]],
            "a partition is resumed only once it leaves the target",
        );
    }

    #[test]
    fn deserializes_a_full_shuffler_envelope() {
        let value = json!({
            "team_id": 42,
            "person_id": "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee",
            "distinct_id": "[email protected]",
            "uuid": "0192f00d-f00d-f00d-f00d-f00df00df00d",
            "event": "$pageview",
            "timestamp": "2026-05-26 12:34:56.789000",
            "properties": "{\"$browser\":\"Chrome\"}",
            "person_properties": "{\"email\":\"[email protected]\"}",
            "elements_chain": "a:href=\"/x\"",
            "source_offset": 12345,
            "source_partition": 17,
        });

        let event: CohortStreamEvent = serde_json::from_value(value).unwrap();
        assert_eq!(event.team_id, 42);
        assert_eq!(event.person_id, "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee");
        assert_eq!(event.event, "$pageview");
        assert_eq!(
            event.properties.as_deref(),
            Some("{\"$browser\":\"Chrome\"}")
        );
        assert_eq!(event.source_offset, 12345);
        assert_eq!(event.source_partition, 17);
    }

    #[test]
    fn null_optional_payloads_deserialize_to_none() {
        let value = json!({
            "team_id": 1,
            "person_id": "p",
            "distinct_id": "d",
            "uuid": "u",
            "event": "$pageview",
            "timestamp": "2026-05-26 12:34:56.789000",
            "properties": null,
            "person_properties": null,
            "elements_chain": null,
            "source_offset": 0,
            "source_partition": 0,
        });

        let event: CohortStreamEvent = serde_json::from_value(value).unwrap();
        assert!(event.properties.is_none());
        assert!(event.person_properties.is_none());
        assert!(event.elements_chain.is_none());
    }

    #[test]
    fn shuffler_envelope_without_redirected_from_deserializes_to_none() {
        let value = json!({
            "team_id": 7,
            "person_id": "p",
            "distinct_id": "d",
            "uuid": "u",
            "event": "$pageview",
            "timestamp": BASE_TS,
            "properties": null,
            "person_properties": null,
            "elements_chain": null,
            "source_offset": 0,
            "source_partition": 0,
        });
        let event: CohortStreamEvent = serde_json::from_slice(&serde_json::to_vec(&value).unwrap())
            .expect("an envelope without redirected_from deserializes");
        assert!(event.redirected_from.is_none());
        assert_eq!(event.redirect_hops, 0);
    }

    #[test]
    fn redirected_from_round_trips_for_the_c2_re_produce() {
        let event = matching_event(Uuid::from_u128(1), 3, 9);
        let with_origin = CohortStreamEvent {
            redirected_from: Some("01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee".to_string()),
            ..event
        };
        let bytes = serde_json::to_vec(&with_origin).unwrap();
        let decoded: CohortStreamEvent = serde_json::from_slice(&bytes).unwrap();
        assert_eq!(
            decoded.redirected_from.as_deref(),
            Some("01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee"),
        );
    }

    #[test]
    fn envelope_round_trips_through_wire_bytes() {
        let envelope = json!({
            "team_id": 7,
            "person_id": "01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee",
            "distinct_id": "d",
            "uuid": "u",
            "event": "$pageview",
            "timestamp": BASE_TS,
            "properties": "{}",
            "person_properties": null,
            "elements_chain": null,
            "source_offset": 99,
            "source_partition": 3,
        });
        let bytes = serde_json::to_vec(&envelope).unwrap();

        let event: CohortStreamEvent = serde_json::from_slice(&bytes).unwrap();
        assert_eq!(event.team_id, 7);
        assert_eq!(event.source_offset, 99);
        assert_eq!(event.source_partition, 3);
        assert!(event.person_properties.is_none());
    }

    fn temp_store() -> (TempDir, CohortStore) {
        let dir = TempDir::new().unwrap();
        let config = StoreConfig {
            path: dir.path().join("db"),
            ..StoreConfig::default()
        };
        let store = CohortStore::open(&config).expect("open store");
        (dir, store)
    }

    fn behavioral_catalog() -> Arc<CatalogHandle> {
        let behavioral_leaf = json!({
            "type": "behavioral",
            "value": "performed_event",
            "key": "$pageview",
            "time_value": 7,
            "time_interval": "day",
            "conditionHash": "0123456789abcdef",
            "bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
        });
        let cohort = json!({ "properties": { "type": "AND", "values": [behavioral_leaf] } });
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(CohortId(1), TeamId(TEAM), &cohort)
            .expect("add cohort");
        let filters = builder.freeze(UTC);
        Arc::new(CatalogHandle::from_catalog(FilterCatalog::from_teams([(
            TeamId(TEAM),
            filters,
        )])))
    }

    fn dispatcher_with(store: &CohortStore, catalog: Arc<CatalogHandle>) -> EventDispatcher {
        dispatcher_and_sink(store, catalog).0
    }

    fn dispatcher_and_sink(
        store: &CohortStore,
        catalog: Arc<CatalogHandle>,
    ) -> (EventDispatcher, Arc<CaptureSink>) {
        let (dispatcher, sink, _, _) = dispatcher_full(
            store,
            catalog,
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );
        (dispatcher, sink)
    }

    fn dispatcher_full(
        store: &CohortStore,
        catalog: Arc<CatalogHandle>,
        sink: CaptureSink,
        transfer_sink: CaptureTransferSink,
    ) -> (
        EventDispatcher,
        Arc<CaptureSink>,
        Arc<CaptureTransferSink>,
        Arc<MergeWorkerDeps>,
    ) {
        let sink = Arc::new(sink);
        let transfer_sink = Arc::new(transfer_sink);
        let transfer_sink_dyn: Arc<dyn crate::producer::TransferSink> = transfer_sink.clone();
        let reconcile = crate::workers::ReconcileDeps {
            enabled: true,
            ..crate::workers::ReconcileDeps::default()
        };
        let merge = Arc::new(MergeWorkerDeps {
            transfer_sink: transfer_sink_dyn,
            stream_event_sink: Arc::new(CaptureStreamEventSink::new()),
            merge_tracker: Arc::new(OffsetTracker::new()),
            transfer_tracker: Arc::new(OffsetTracker::new()),
            retry: TransferRetryPolicy::default(),
            gc_scan_limit: crate::workers::DEFAULT_MERGE_GC_SCAN_LIMIT,
            stage2_orphan_gc_enabled: true,
            cascade_sink: Arc::new(crate::producer::CaptureCascadeSink::new()),
            cascade_tracker: Arc::new(OffsetTracker::new()),
            cascade: crate::workers::CascadeConfig::default(),
            partition_count: COHORT_PARTITION_COUNT,
            seed_tile_sink: Arc::new(crate::producer::CaptureSeedTileSink::new()),
            seed_tracker: Arc::new(crate::partitions::offset_tracker::OffsetTracker::new()),
            live_watermarks: Arc::new(crate::partitions::watermarks::LiveWatermarks::new()),
            // The dispatch tests exercise cross-partition register transfer end to end.
            register_transfer_enabled: true,
            reconcile,
            person_seed: crate::workers::PersonSeedDeps::default(),
            seed_budget: crate::workers::seed_run::RunBudget::default(),
        });
        let dispatcher = EventDispatcher::new(
            PartitionRouter::new(64),
            Arc::new(OffsetTracker::new()),
            test_handle(store),
            catalog,
            sink.clone(),
            merge.clone(),
        );
        (dispatcher, sink, transfer_sink, merge)
    }

    fn behavioral_lsk(catalog: &CatalogHandle) -> LeafStateKey {
        let snapshot = catalog.load();
        let team = snapshot.team(TeamId(TEAM)).expect("team in catalog");
        team.by_condition_to_lsk[&BEHAVIORAL_HASH][0]
    }

    fn person(n: u128) -> Uuid {
        Uuid::from_u128(n)
    }

    fn matching_event(
        person: Uuid,
        source_partition: i32,
        source_offset: i64,
    ) -> CohortStreamEvent {
        CohortStreamEvent {
            team_id: TEAM,
            person_id: person.to_string(),
            distinct_id: "d".to_string(),
            uuid: Uuid::from_u128(0xE0_0000 + source_offset as u128).to_string(),
            event: "$pageview".to_string(),
            timestamp: BASE_TS.to_string(),
            properties: Some("{}".to_string()),
            person_properties: None,
            elements_chain: None,
            source_offset,
            source_partition,
            redirected_from: None,
            redirect_hops: 0,
        }
    }

    fn consumed(person: Uuid, topic_partition: i32, topic_offset: i64) -> ConsumedEvent {
        ConsumedEvent {
            event: matching_event(person, topic_partition, topic_offset),
            partition: topic_partition,
            offset: topic_offset,
            broker_ts_ms: None,
        }
    }

    fn behavioral_state(
        store: &CohortStore,
        partition_id: u16,
        person: Uuid,
        lsk: LeafStateKey,
    ) -> Option<Stage1State> {
        let key = BehavioralKey::new(partition_id, TEAM as u64, person, lsk);
        store
            .get_behavioral(&key)
            .unwrap()
            .map(|bytes| StatefulRecord::decode(&bytes).unwrap().state)
    }

    fn seed_slice(store: &CohortStore, partition: u16, lsk: LeafStateKey) {
        store
            .write_batch(|b| {
                b.put::<Behavioral>(
                    &BehavioralKey::new(partition, TEAM as u64, person(1), lsk),
                    b"state",
                )
            })
            .unwrap();
    }

    fn slice_present(store: &CohortStore, partition: u16, lsk: LeafStateKey) -> bool {
        store
            .get_behavioral(&BehavioralKey::new(partition, TEAM as u64, person(1), lsk))
            .unwrap()
            .is_some()
    }

    #[tokio::test]
    async fn the_boot_sweep_wipes_only_the_slices_another_pod_owns() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let lsk = behavioral_lsk(&catalog);
        let dispatcher = dispatcher_with(&store, catalog);
        for partition in 0..4u16 {
            seed_slice(&store, partition, lsk);
        }
        let pod_zero_of_two =
            PartitionOwnership::new(NonZeroU16::new(2).unwrap(), PodOrdinal::STANDALONE, 4)
                .unwrap();

        assert_eq!(dispatcher.reclaim_unowned_slices(&pod_zero_of_two).await, 2);

        assert_eq!(
            (0..4u16)
                .map(|partition| slice_present(&store, partition, lsk))
                .collect::<Vec<_>>(),
            [true, false, true, false],
        );
        assert_eq!(dispatcher.reclaim_unowned_slices(&pod_zero_of_two).await, 0);
    }

    #[tokio::test]
    async fn dispatch_lazily_spawns_one_worker_per_partition_and_marks_next_offsets() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let lsk = behavioral_lsk(&catalog);
        let dispatcher = dispatcher_with(&store, catalog);

        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);

        let batch = vec![
            consumed(person(1), 0, 10),
            consumed(person(2), 0, 11),
            consumed(person(3), 1, 5),
        ];
        dispatcher.dispatch(batch).await;

        assert_eq!(dispatcher.workers.len(), 2);

        let tracker = dispatcher.shutdown().await;

        let committable = tracker.committable_offsets();
        assert_eq!(committable.get(&0), Some(&12));
        assert_eq!(committable.get(&1), Some(&6));

        assert!(
            matches!(
                behavioral_state(&store, 0, person(1), lsk),
                Some(Stage1State::BehavioralSingle {
                    has_match: true,
                    ..
                })
            ),
            "person 1 should have entered under partition 0",
        );
        assert!(
            matches!(
                behavioral_state(&store, 0, person(2), lsk),
                Some(Stage1State::BehavioralSingle {
                    has_match: true,
                    ..
                })
            ),
            "person 2 should have entered under partition 0",
        );
        assert!(
            matches!(
                behavioral_state(&store, 1, person(3), lsk),
                Some(Stage1State::BehavioralSingle {
                    has_match: true,
                    ..
                })
            ),
            "person 3 should have entered under partition 1",
        );
    }

    #[tokio::test]
    async fn dispatch_reuses_an_existing_worker_for_a_known_partition() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.assign_partition(0);

        dispatcher.dispatch(vec![consumed(person(1), 0, 1)]).await;
        assert_eq!(dispatcher.workers.len(), 1);

        dispatcher.dispatch(vec![consumed(person(2), 0, 2)]).await;
        assert_eq!(dispatcher.workers.len(), 1);

        let tracker = dispatcher.shutdown().await;
        assert_eq!(tracker.committable_offsets().get(&0), Some(&3));
    }

    #[tokio::test]
    async fn dispatch_route_error_does_not_advance_the_offset() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.assign_partition(9);
        dispatcher.ensure_worker(9);
        dispatcher.router.remove_partition(9);

        dispatcher.dispatch(vec![consumed(person(1), 9, 100)]).await;

        let tracker = dispatcher.shutdown().await;
        assert_eq!(
            tracker.committable_offsets().get(&9),
            None,
            "a route error leaves the offset unmarked for Kafka to replay",
        );
        assert_eq!(tracker.partition_count(), 1);
    }

    /// Re-acquiring a partition clears any stale watermark, so every tenure starts fail-closed.
    #[tokio::test]
    async fn assign_partition_resets_the_live_watermark_so_a_new_tenure_is_fail_closed() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.merge_deps().live_watermarks.observe(5, 1_000);
        dispatcher.assign_partition(5);

        assert_eq!(dispatcher.merge_deps().live_watermarks.get(5), None);
        let _tracker = dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn dispatch_empty_batch_is_a_noop() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.dispatch(vec![]).await;

        assert_eq!(dispatcher.workers.len(), 0);
        assert!(dispatcher.tracker().committable_offsets().is_empty());
        let _tracker = dispatcher.shutdown().await;
    }

    /// A dispatcher whose router uses `buffer` channel slots and no real worker, so a pre-filled
    /// channel exercises the `try_send`-full path.
    fn dispatcher_with_buffer(
        store: &CohortStore,
        catalog: Arc<CatalogHandle>,
        buffer: usize,
    ) -> EventDispatcher {
        let sink: Arc<dyn MembershipSink> = Arc::new(CaptureSink::new());
        EventDispatcher::new(
            PartitionRouter::new(buffer),
            Arc::new(OffsetTracker::new()),
            test_handle(store),
            catalog,
            sink,
            MergeWorkerDeps::capture(),
        )
    }

    /// Like [`dispatcher_with_buffer`] but with a per-partition event-intake `cap` and a seed-lane
    /// capacity, so a test can trip either lane's budget while the mpsc slots stay free.
    fn dispatcher_with_intake_cap(
        store: &CohortStore,
        catalog: Arc<CatalogHandle>,
        buffer: usize,
        cap: usize,
        seed_cap: usize,
    ) -> EventDispatcher {
        let sink: Arc<dyn MembershipSink> = Arc::new(CaptureSink::new());
        EventDispatcher::new(
            PartitionRouter::with_intake_cap(
                buffer,
                cap,
                NonZeroUsize::new(seed_cap).expect("a test seed cap is non-zero"),
            ),
            Arc::new(OffsetTracker::new()),
            test_handle(store),
            catalog,
            sink,
            MergeWorkerDeps::capture(),
        )
    }

    fn held_offsets(messages: &[ShuffleMessage]) -> Vec<i64> {
        messages
            .iter()
            .filter_map(ShuffleMessage::event_offset)
            .collect()
    }

    fn consumed_seed(seed_person: Uuid, partition: i32, offset: i64) -> ConsumedSeed {
        use cohort_core::seed::{ClaimEpoch, ConditionHash, RunId, SChunkMs, SeedTile};
        ConsumedSeed {
            work: crate::consumers::seeds::SeedWork::Tile(SeedTile::new(
                TeamId(TEAM),
                seed_person,
                ConditionHash::parse("0123456789abcdef").unwrap(),
                std::num::NonZeroU32::new(1).unwrap(),
                20_614,
                SChunkMs(1_700_000_000_000),
                RunId(Uuid::nil()),
                ClaimEpoch(1),
            )),
            partition,
            offset,
            broker_ts_ms: None,
        }
    }

    fn held_seed_offsets(seeds: &[ConsumedSeed]) -> Vec<i64> {
        seeds.iter().map(|seed| seed.offset).collect()
    }

    #[tokio::test]
    async fn dispatch_seeds_raises_only_the_seed_ceiling_from_its_own_offsets() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with_buffer(&store, behavioral_catalog(), 16);
        dispatcher.assign_partition(0);
        // Hold the inbox so no worker drains; the ceilings are the observable.
        let mut inbox = dispatcher.router.add_partition(0).unwrap();

        let full = dispatcher.dispatch_seeds(vec![
            consumed_seed(person(1), 0, 3),
            consumed_seed(person(2), 0, 5),
            // Unowned partition: dropped for replay, never held.
            consumed_seed(person(3), 9, 1),
        ]);
        assert!(full.is_empty(), "both owned seeds landed");

        // Seeds never ride the live lane, so the dispatcher must have raised the seed ceiling
        // itself.
        let seed_tracker = &dispatcher.merge_deps().seed_tracker;
        assert_eq!(
            seed_tracker.mark_processed(0, 6),
            MarkOutcome::WithinDispatch
        );
        assert_eq!(seed_tracker.committable_offsets().get(&0), Some(&6));
        assert_eq!(
            seed_tracker.mark_processed(0, 7),
            MarkOutcome::CappedAheadOfDispatch,
            "the ceiling is exactly the dispatched max + 1",
        );

        assert_eq!(
            dispatcher.tracker().mark_processed(0, 1),
            MarkOutcome::CappedAheadOfDispatch,
        );
        assert!(dispatcher.tracker().committable_offsets().is_empty());

        let mut landed = Vec::new();
        assert_eq!(
            inbox.seeds.recv_many(&mut landed, 8).now_or_never(),
            Some(2)
        );
        assert_eq!(held_seed_offsets(&landed), vec![3, 5]);
        assert!(
            inbox.live.try_recv().is_err(),
            "nothing reached the live lane",
        );
    }

    #[tokio::test]
    async fn dispatch_seeds_lands_while_the_live_intake_is_full() {
        let (_dir, store) = temp_store();
        // One-event live budget, a roomy seed lane: the two must not share a budget.
        let dispatcher = dispatcher_with_intake_cap(&store, behavioral_catalog(), 16, 1, 16);
        dispatcher.assign_partition(0);
        let mut inbox = dispatcher.router.add_partition(0).unwrap();
        let no_held = HashSet::new();

        assert!(dispatcher
            .dispatch_events_nonblocking(vec![consumed(person(1), 0, 104)], &no_held)
            .is_empty());
        assert!(
            dispatcher
                .dispatch_events_nonblocking(vec![consumed(person(2), 0, 105)], &no_held)
                .contains_key(&0),
            "the live lane is at its event ceiling",
        );

        assert!(
            dispatcher
                .dispatch_seeds(vec![consumed_seed(person(3), 0, 10)])
                .is_empty(),
            "the seed lane is unaffected by a full live intake",
        );
        let seed_tracker = &dispatcher.merge_deps().seed_tracker;
        assert_eq!(
            seed_tracker.mark_processed(0, 11),
            MarkOutcome::WithinDispatch,
        );
        assert_eq!(seed_tracker.committable_offsets().get(&0), Some(&11));

        let mut landed = Vec::new();
        assert_eq!(
            inbox.seeds.recv_many(&mut landed, 8).now_or_never(),
            Some(1)
        );
        assert_eq!(held_seed_offsets(&landed), vec![10]);
    }

    #[tokio::test]
    async fn dispatch_seeds_holds_the_suffix_on_the_seed_cap_and_raises_the_ceiling_only_for_what_landed(
    ) {
        let (_dir, store) = temp_store();
        // A one-seed lane, so the second seed of the sub-batch is refused.
        let dispatcher = dispatcher_with_intake_cap(&store, behavioral_catalog(), 16, 16, 1);
        dispatcher.assign_partition(0);
        let _inbox = dispatcher.router.add_partition(0).unwrap();

        let mut full = dispatcher.dispatch_seeds(vec![
            consumed_seed(person(1), 0, 10),
            consumed_seed(person(2), 0, 11),
        ]);
        let held = full.remove(&0).expect("partition 0 held on the seed cap");
        assert_eq!(held_seed_offsets(&held), vec![11]);

        let seed_tracker = &dispatcher.merge_deps().seed_tracker;
        assert_eq!(
            seed_tracker.mark_processed(0, 12),
            MarkOutcome::CappedAheadOfDispatch,
            "a held seed's offset stays above the ceiling, so it can never commit",
        );
        assert_eq!(seed_tracker.committable_offsets().get(&0), Some(&11));
    }

    #[tokio::test]
    async fn nonblocking_dispatch_holds_a_full_channel_and_advances_the_offset_only_after_flush() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with_buffer(&store, behavioral_catalog(), 1);
        dispatcher.assign_partition(0);
        // Hold the receiver so nothing drains; ensure_worker reuses the registered channel and spawns
        // no worker, leaving this rx the only consumer.
        let mut rx = dispatcher.router.add_partition(0).unwrap();
        let no_held = HashSet::new();

        // Batch A fills the one slot and raises the ceiling to 105.
        assert!(dispatcher
            .dispatch_events_nonblocking(vec![consumed(person(1), 0, 104)], &no_held)
            .is_empty());
        // Batch B finds the channel full: held verbatim, ceiling unchanged.
        let mut full =
            dispatcher.dispatch_events_nonblocking(vec![consumed(person(2), 0, 105)], &no_held);
        let held = full.remove(&0).expect("partition 0 was held");
        assert_eq!(held_offsets(&held), vec![105]);
        assert!(
            dispatcher.workers.is_empty(),
            "no worker drained the channel"
        );

        // The hole at 105 can't be committed past: a worker marking beyond the ceiling clamps to it.
        assert_eq!(
            dispatcher.tracker().mark_processed(0, 106),
            MarkOutcome::CappedAheadOfDispatch,
        );
        assert_eq!(
            dispatcher.tracker().committable_offsets().get(&0),
            Some(&105),
            "committable pinned at the sent ceiling while 105 is held",
        );

        // Drain A; the retry-flush now sends B and the ceiling advances past it.
        let _a = rx.live.recv().await.unwrap();
        assert!(
            dispatcher.redispatch_held(vec![(0, held)]).is_empty(),
            "the holdover flushed once the channel had room",
        );
        assert_eq!(
            dispatcher.tracker().mark_processed(0, 106),
            MarkOutcome::WithinDispatch,
        );
        assert_eq!(
            dispatcher.tracker().committable_offsets().get(&0),
            Some(&106)
        );
    }

    #[tokio::test]
    async fn nonblocking_dispatch_holds_on_the_event_cap_while_slots_stay_free() {
        let (_dir, store) = temp_store();
        // 16 mpsc slots but a one-event budget: the event cap, not the slots, is what holds batch B.
        let dispatcher = dispatcher_with_intake_cap(&store, behavioral_catalog(), 16, 1, 16);
        dispatcher.assign_partition(0);
        let mut rx = dispatcher.router.add_partition(0).unwrap();
        let no_held = HashSet::new();

        // Batch A fills the one-event budget and raises the ceiling to 105.
        assert!(dispatcher
            .dispatch_events_nonblocking(vec![consumed(person(1), 0, 104)], &no_held)
            .is_empty());
        // Batch B is refused by the event cap even though 15 slots remain free.
        let mut full =
            dispatcher.dispatch_events_nonblocking(vec![consumed(person(2), 0, 105)], &no_held);
        let held = full.remove(&0).expect("partition 0 held on the event cap");
        assert_eq!(held_offsets(&held), vec![105]);
        assert_eq!(
            dispatcher.tracker().mark_processed(0, 106),
            MarkOutcome::CappedAheadOfDispatch,
            "the held event stays above the dispatch ceiling",
        );

        // Drain A and step past it so the budget frees (release lands on the next recv), then B flushes.
        let _a = rx.live.recv().await.unwrap();
        assert!(
            rx.live.try_recv().is_err(),
            "channel drained; the recv released A's budget",
        );
        assert!(
            dispatcher.redispatch_held(vec![(0, held)]).is_empty(),
            "the held batch flushes once the event budget frees",
        );
        assert_eq!(
            dispatcher.tracker().mark_processed(0, 106),
            MarkOutcome::WithinDispatch,
        );
    }

    #[tokio::test]
    async fn nonblocking_dispatch_defers_events_for_a_held_partition_even_with_room() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with_buffer(&store, behavioral_catalog(), 16);
        dispatcher.assign_partition(0);
        let mut rx = dispatcher.router.add_partition(0).unwrap();

        // Partition 0 is already held: its fresh events queue behind the holdover rather than racing
        // onto the channel, even with room.
        let held: HashSet<i32> = [0].into_iter().collect();
        let mut full =
            dispatcher.dispatch_events_nonblocking(vec![consumed(person(1), 0, 200)], &held);

        assert_eq!(held_offsets(&full.remove(&0).unwrap()), vec![200]);
        assert!(
            rx.live.try_recv().is_err(),
            "nothing was sent to the channel"
        );
        assert!(
            !dispatcher.tracker().committable_offsets().contains_key(&0),
            "a deferred event never raised the ceiling",
        );
    }

    #[tokio::test]
    async fn nonblocking_dispatch_returns_promptly_when_the_channel_is_saturated() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with_buffer(&store, behavioral_catalog(), 1);
        dispatcher.assign_partition(0);
        let _rx = dispatcher.router.add_partition(0).unwrap();
        let no_held = HashSet::new();

        dispatcher.dispatch_events_nonblocking(vec![consumed(person(1), 0, 0)], &no_held);
        // The second dispatch onto the saturated channel returns the holdover synchronously instead of
        // awaiting a drain.
        let full =
            dispatcher.dispatch_events_nonblocking(vec![consumed(person(2), 0, 1)], &no_held);
        assert!(
            full.contains_key(&0),
            "the saturated partition's events are held, never blocked on",
        );
    }

    /// A worker resolves its slice coverage first, so a processed offset proves it resolved.
    async fn processed_through(dispatcher: &EventDispatcher, partition: i32, next: i64) {
        let start = Instant::now();
        while dispatcher.tracker.committable_offsets().get(&partition) != Some(&next) {
            assert!(
                start.elapsed() < Duration::from_secs(10),
                "offset {next} on partition {partition} was never processed",
            );
            tokio::time::sleep(Duration::from_millis(10)).await;
        }
    }

    #[tokio::test]
    async fn a_spawn_resumes_recorded_coverage_and_begins_a_new_slice_at_its_assignment() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());
        // Recorded by an earlier tenure.
        let earlier = SliceCoverage::Since(CoverageStartMs(1));
        store
            .write_batch(|b| b.put::<SliceCoverages>(&SliceCoverageKey(0), &earlier.encode()))
            .unwrap();

        dispatcher.assign_partition(0);
        dispatcher.dispatch(vec![consumed(person(1), 0, 10)]).await;
        processed_through(&dispatcher, 0, 11).await;
        assert_eq!(
            store.slice_coverage(0).unwrap(),
            Some(earlier),
            "a spawn resumes the recorded coverage instead of beginning the slice again",
        );

        let before_assignment = CoverageStartMs::now();
        dispatcher.assign_partition(1);
        let assigned = CoverageStartMs::now();
        // Spawn in a later millisecond, so a slice begun at spawn fails the bound below.
        while CoverageStartMs::now() <= assigned {
            tokio::task::yield_now().await;
        }
        dispatcher.dispatch(vec![consumed(person(2), 1, 20)]).await;
        processed_through(&dispatcher, 1, 21).await;
        assert!(
            matches!(
                store.slice_coverage(1).unwrap(),
                Some(SliceCoverage::Since(since))
                    if before_assignment <= since && since <= assigned
            ),
            "a slice without a record begins at its assignment rather than its first delivery",
        );

        dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn the_drain_signal_waits_for_every_worker_and_records_the_ownership_it_drained_under() {
        let (_dir, store) = temp_store();
        let sink = BlockingMembershipSink::new();
        let dispatcher = Arc::new(EventDispatcher::new(
            PartitionRouter::new(64),
            Arc::new(OffsetTracker::new()),
            test_handle(&store),
            behavioral_catalog(),
            sink.clone(),
            MergeWorkerDeps::capture(),
        ));
        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);
        dispatcher.dispatch(vec![consumed(person(1), 0, 10)]).await;
        sink.entered.notified().await;

        let drain = tokio::spawn({
            let dispatcher = dispatcher.clone();
            async move { dispatcher.shutdown().await }
        });
        assert!(matches!(
            dispatcher
                .wait_for_worker_drain(Duration::from_millis(100))
                .await,
            DrainWait::TimedOut,
        ));

        sink.release.notify_one();
        drain.await.unwrap();
        let DrainWait::Drained(drained) = dispatcher
            .wait_for_worker_drain(Duration::from_secs(10))
            .await
        else {
            panic!("the drain finished once the blocked produce was released");
        };
        assert_eq!(drained.partitions(), &HashSet::from([0, 1]));
        assert!(
            matches!(
                dispatcher.wait_for_worker_drain(Duration::ZERO).await,
                DrainWait::Drained(_)
            ),
            "the final checkpoint waits with no budget, so a finished drain must answer at once",
        );

        let follower_tracker = OffsetTracker::new();
        for partition in [0, 1, 2] {
            follower_tracker.mark_dispatched(partition, 6);
            assert_eq!(
                follower_tracker.mark_processed(partition, 6),
                MarkOutcome::WithinDispatch
            );
        }
        assert_eq!(
            crate::consumers::merges::committable_after_drain(
                &follower_tracker,
                &dispatcher,
                HashSet::new(),
                "follower_topic",
            )
            .await,
            HashMap::from([(0, 6), (1, 6)]),
            "a follower commits for the partitions drained, not the set it saw at shutdown",
        );
    }

    #[tokio::test]
    async fn finishing_boot_rebuilds_every_owned_partition_before_the_gate_opens() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());
        dispatcher.enable_durable_restore();
        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);
        let readiness = dispatcher.readiness().clone();
        assert_eq!(readiness.check(), Err(NotReady::Booting));

        assert_eq!(dispatcher.finish_boot(), 2);
        assert_eq!(
            readiness.check(),
            Err(NotReady::Rebuilding { partitions: 2 }),
            "both partitions rebuild although neither received a message",
        );

        let start = Instant::now();
        while readiness.check().is_err() {
            assert!(
                start.elapsed() < Duration::from_secs(10),
                "each rebuild releases the gate when its scan ends, got {:?}",
                readiness.check(),
            );
            tokio::time::sleep(Duration::from_millis(10)).await;
        }
        dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn owned_committable_offsets_excludes_unowned_partitions() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.assign_partition(1);
        for (partition, next) in [(0, 11), (1, 21)] {
            dispatcher.tracker().mark_dispatched(partition, next);
            let _ = dispatcher.tracker().mark_processed(partition, next);
        }

        assert_eq!(
            dispatcher.owned_committable_offsets(),
            HashMap::from([(1, 21)])
        );
    }

    #[tokio::test]
    async fn route_sweep_delivers_the_cutoff_to_each_owned_worker_once_the_catalog_loads() {
        let (_dir, store) = temp_store();
        let catalog = Arc::new(CatalogHandle::new());
        let dispatcher = dispatcher_with(&store, catalog.clone());

        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);
        let mut rx0 = dispatcher.router.add_partition(0).unwrap();
        let mut rx1 = dispatcher.router.add_partition(1).unwrap();

        let cutoff = 1_700_000_000_000;
        dispatcher.route_sweep(cutoff).await;
        for rx in [&mut rx0, &mut rx1] {
            assert!(
                rx.live.try_recv().is_err(),
                "a sweep before the first catalog load would drop every restored deadline",
            );
        }

        catalog.publish(FilterCatalog::new());
        dispatcher.route_sweep(cutoff).await;

        for rx in [&mut rx0, &mut rx1] {
            let batch = rx.live.recv().await.expect("a sweep was routed");
            assert_eq!(batch.len(), 1);
            match &batch[0] {
                ShuffleMessage::Sweep { due_before_ms } => assert_eq!(*due_before_ms, cutoff),
                other => panic!("expected Sweep, got {other:?}"),
            }
        }
    }

    #[tokio::test]
    async fn route_reconcile_drain_delivers_one_tick_to_each_active_worker() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);
        let mut rx0 = dispatcher.router.add_partition(0).unwrap();
        let mut rx1 = dispatcher.router.add_partition(1).unwrap();

        dispatcher.route_reconcile_drain().await;

        for receiver in [&mut rx0, &mut rx1] {
            let batch = receiver
                .live
                .recv()
                .await
                .expect("a reconcile tick was routed");
            assert!(matches!(batch.as_slice(), [ShuffleMessage::ReconcileDrain]));
        }
    }

    #[tokio::test]
    async fn route_sweep_is_benign_for_workerless_or_no_owned_partitions() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.route_sweep(123).await;
        dispatcher.assign_partition(0);
        dispatcher.route_sweep(123).await;
        assert_eq!(dispatcher.workers.len(), 0, "a sweep never spawns a worker");
        assert_eq!(
            dispatcher.router.partition_count(),
            0,
            "no channel registered"
        );
    }

    #[tokio::test]
    async fn maintenance_ticks_skip_owned_partitions_without_a_worker() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        // Three owned partitions, but only 0 and 2 ever spawned a worker. Partition 1 is owned and
        // worker-less — the steady state of an idle partition that never received an event, which is
        // what produced the `no_worker` route-drop noise.
        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);
        dispatcher.assign_partition(2);
        let mut rx0 = dispatcher.router.add_partition(0).unwrap();
        let mut rx2 = dispatcher.router.add_partition(2).unwrap();

        let mut tickable = dispatcher.tickable_partitions();
        tickable.sort_unstable();
        assert_eq!(
            tickable,
            vec![0, 2],
            "a tick targets only owned partitions with a live worker, never the worker-less one",
        );

        // The fan-out reaches the worker-bearing partitions and silently skips the worker-less one
        // (no `no_worker` drop): only 0 and 2 receive the sweep.
        dispatcher.route_sweep(777).await;
        for rx in [&mut rx0, &mut rx2] {
            let batch = rx
                .live
                .recv()
                .await
                .expect("a worker-bearing partition got the sweep");
            assert_eq!(batch.len(), 1);
            match &batch[0] {
                ShuffleMessage::Sweep { due_before_ms } => assert_eq!(*due_before_ms, 777),
                other => panic!("expected Sweep, got {other:?}"),
            }
        }
    }

    #[tokio::test]
    async fn ensure_worker_does_not_spawn_for_an_unowned_partition() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.ensure_worker(5);
        assert_eq!(
            dispatcher.workers.len(),
            0,
            "no worker for an unowned partition",
        );
        assert_eq!(
            dispatcher.router.partition_count(),
            0,
            "no channel registered",
        );

        dispatcher.assign_partition(5);
        dispatcher.ensure_worker(5);
        assert_eq!(dispatcher.workers.len(), 1);
        assert_eq!(dispatcher.router.partition_count(), 1);

        let _tracker = dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn dispatch_routes_owned_and_drops_unowned_without_raising_its_ceiling() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let lsk = behavioral_lsk(&catalog);
        let dispatcher = dispatcher_with(&store, catalog);

        dispatcher.assign_partition(0);

        dispatcher
            .dispatch(vec![consumed(person(1), 0, 10), consumed(person(2), 1, 20)])
            .await;

        assert_eq!(dispatcher.workers.len(), 1);
        assert!(dispatcher.workers.contains_key(&0));
        assert!(!dispatcher.workers.contains_key(&1));
        assert_eq!(
            dispatcher.tracker().partition_count(),
            1,
            "the dropped partition's dispatch ceiling was never raised",
        );

        let tracker = dispatcher.shutdown().await;
        assert_eq!(
            tracker.committable_offsets().get(&0),
            Some(&11),
            "the owned partition advanced",
        );
        assert!(behavioral_state(&store, 0, person(1), lsk).is_some());
        assert!(
            behavioral_state(&store, 1, person(2), lsk).is_none(),
            "the unowned partition wrote no state",
        );
    }

    #[tokio::test]
    async fn shutdown_drains_a_late_dispatch_without_stranding_its_worker() {
        let (_dir, store) = temp_store();
        let (dispatcher, sink) = dispatcher_and_sink(&store, behavioral_catalog());

        dispatcher.assign_partition(0);
        dispatcher.dispatch(vec![consumed(person(1), 0, 10)]).await;
        assert_eq!(
            dispatcher.workers.len(),
            1,
            "the late dispatch spawned a worker",
        );

        let tracker = dispatcher.shutdown().await;

        assert_eq!(
            dispatcher.workers.len(),
            0,
            "shutdown joined the late worker rather than stranding it",
        );
        assert_eq!(
            tracker.committable_offsets().get(&0),
            Some(&11),
            "its produced tail's offset is committable, so a crash here would replay",
        );
        assert_eq!(sink.changes().len(), 1, "the worker flushed before exiting");
    }

    const MERGED_AT: i64 = 1_716_800_000_000;

    fn composable_catalog() -> Arc<CatalogHandle> {
        let behavioral_leaf = |days: i64| {
            json!({
                "type": "behavioral",
                "value": "performed_event",
                "key": "$pageview",
                "time_value": days,
                "time_interval": "day",
                "conditionHash": "0123456789abcdef",
                "bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
            })
        };
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(
                CohortId(1),
                TeamId(TEAM),
                &json!({ "properties": { "type": "AND", "values": [behavioral_leaf(7)] } }),
            )
            .expect("add single-leaf cohort");
        builder
            .add_cohort(
                CohortId(2),
                TeamId(TEAM),
                &json!({ "properties": { "type": "AND", "values": [behavioral_leaf(7), behavioral_leaf(30)] } }),
            )
            .expect("add composable cohort");
        Arc::new(CatalogHandle::from_catalog(FilterCatalog::from_teams([(
            TeamId(TEAM),
            builder.freeze(UTC),
        )])))
    }

    fn team_without_cohorts_catalog() -> Arc<CatalogHandle> {
        Arc::new(CatalogHandle::from_catalog(FilterCatalog::for_processing(
            TeamAllowlist::Only(HashSet::from([TEAM])),
            [],
        )))
    }

    fn person_hashing_to(partition: u16, start: u128) -> Uuid {
        (start..start + 100_000)
            .map(Uuid::from_u128)
            .find(|p| partition_of(TeamId(TEAM), p, COHORT_PARTITION_COUNT) as u16 == partition)
            .expect("some person hashes to the partition")
    }

    fn person_hashing_away_from(partition: u16, start: u128) -> Uuid {
        (start..start + 100_000)
            .map(Uuid::from_u128)
            .find(|p| partition_of(TeamId(TEAM), p, COHORT_PARTITION_COUNT) as u16 != partition)
            .expect("some person hashes to another partition")
    }

    fn consumed_merge(p_old: Uuid, p_new: Uuid, partition: i32, offset: i64) -> ConsumedMerge {
        ConsumedMerge {
            event: PersonMergeEvent {
                team_id: TEAM,
                old_person_uuid: p_old,
                new_person_uuid: p_new,
                merged_at_ms: MERGED_AT,
                schema_version: MERGE_EVENT_SCHEMA_VERSION,
            },
            partition,
            offset,
        }
    }

    fn behavioral_match_record() -> StatefulRecord {
        StatefulRecord::new(
            Stage1State::BehavioralSingle {
                has_match: true,
                last_event_at_ms: MERGED_AT - 1_000,
                earliest_eviction_at_ms: i64::MAX,
            },
            AppliedOffsets::default(),
        )
    }

    fn transfer_with(
        p_old: Uuid,
        p_new: Uuid,
        source: (i32, i64),
        leaves: Vec<TransferLeaf>,
    ) -> MergeStateTransfer {
        MergeStateTransfer {
            team_id: TEAM,
            old_person_uuid: p_old,
            new_person_uuid: p_new,
            merged_at_ms: MERGED_AT,
            source_partition: source.0,
            source_offset: source.1,
            leaves,
            membership_registers: vec![],
            forward_hops: 0,

            person_dedup: None,
        }
    }

    fn pending_key_of(partition: i32, p_old: Uuid) -> PendingTransferKey {
        PendingTransferKey {
            partition_id: partition as u16,
            team_id: TEAM as u64,
            old_person: p_old,
        }
    }

    #[tokio::test]
    async fn dispatch_merges_fast_path_composes_membership_and_marks_only_the_merge_tracker() {
        let (_dir, store) = temp_store();
        let (dispatcher, sink, transfer_sink, merge) = dispatcher_full(
            &store,
            composable_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_new = person(2);
        let partition = partition_of(TeamId(TEAM), &p_new, COHORT_PARTITION_COUNT) as i32;
        let p_old = person(1);
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(p_old, partition, 10)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 3)])
            .await;
        let tracker = dispatcher.shutdown().await;

        let mut p_new_changes: Vec<(i32, MembershipStatus)> = sink
            .changes()
            .iter()
            .filter(|change| change.person_id == p_new.to_string())
            .map(|change| (change.cohort_id, change.status))
            .collect();
        p_new_changes.sort_unstable_by_key(|(cohort_id, _)| *cohort_id);
        assert_eq!(
            p_new_changes,
            vec![
                (1, MembershipStatus::Entered),
                (2, MembershipStatus::Entered)
            ],
        );

        assert!(
            transfer_sink.transfers().is_empty(),
            "the fast path produces no transfer",
        );
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&4),
            "the merge offset marks on the merge tracker",
        );
        assert_eq!(
            tracker.committable_offsets().get(&partition),
            Some(&11),
            "the events tracker only ever carries the event's own offset",
        );
        assert_eq!(merge.transfer_tracker.partition_count(), 0);
    }

    #[tokio::test]
    async fn dispatch_merges_slow_path_produces_the_transfer_clears_the_outbox_and_marks() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(p_old, partition, 10)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 5)])
            .await;
        let _tracker = dispatcher.shutdown().await;

        let transfers = transfer_sink.transfers();
        assert_eq!(transfers.len(), 1);
        assert_eq!(transfers[0].new_person_uuid, p_new);
        assert_eq!(
            (transfers[0].source_partition, transfers[0].source_offset),
            (partition, 5),
            "the transfer carries the merge message's source coordinates",
        );
        assert!(
            !transfers[0].leaves.is_empty(),
            "P_old's state was packaged"
        );
        assert!(
            store
                .get_pending_transfer(&pending_key_of(partition, p_old))
                .unwrap()
                .is_none(),
            "the outbox was cleared after the ack",
        );
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&6),
        );
    }

    #[tokio::test(start_paused = true)]
    async fn transfer_produce_retries_inline_on_the_paused_clock_and_succeeds() {
        let wall = std::time::Instant::now();
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::failing_first(2),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(p_old, partition, 0)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 7)])
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert_eq!(
            transfer_sink.transfers().len(),
            1,
            "the third attempt acked"
        );
        assert!(store
            .get_pending_transfer(&pending_key_of(partition, p_old))
            .unwrap()
            .is_none(),);
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&8),
        );
        assert!(
            wall.elapsed() < Duration::from_secs(2),
            "backoff ran on the paused clock, not the wall clock",
        );
    }

    #[tokio::test(start_paused = true)]
    async fn transfer_produce_exhaustion_leaves_the_outbox_pending_and_does_not_wedge() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            catalog.clone(),
            CaptureSink::new(),
            CaptureTransferSink::failing_always(),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(p_old, partition, 0)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 10)])
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert!(transfer_sink.transfers().is_empty());
        let pending = PendingTransfer::decode(
            &store
                .get_pending_transfer(&pending_key_of(partition, p_old))
                .unwrap()
                .expect("the pending entry survives exhaustion"),
        )
        .unwrap();
        assert_eq!(pending.transfer.old_person_uuid, p_old);
        assert!(!pending.transfer.leaves.is_empty());
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            None,
            "exhaustion skips the mark",
        );

        // A later merge on the same partition still processes (no wedge).
        let dispatcher_b = EventDispatcher::new(
            PartitionRouter::new(64),
            Arc::new(OffsetTracker::new()),
            test_handle(&store),
            catalog,
            Arc::new(CaptureSink::new()),
            merge.clone(),
        );
        let p_new_b = person_hashing_to(partition as u16, 200);
        let p_old_b = person(50);
        dispatcher_b.assign_partition(partition);
        dispatcher_b
            .dispatch_merges(vec![consumed_merge(p_old_b, p_new_b, partition, 11)])
            .await;
        let _tracker_b = dispatcher_b.shutdown().await;

        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&12),
            "the subsequent merge processed and marked",
        );
        assert!(
            store
                .get_pending_transfer(&pending_key_of(partition, p_old))
                .unwrap()
                .is_some(),
            "the failed merge's outbox entry is untouched",
        );
    }

    #[tokio::test(start_paused = true)]
    async fn redelivered_merge_re_produces_the_staged_outbox_entry_and_marks() {
        let (_dir, store) = temp_store();
        let budget = TransferRetryPolicy::default().max_retries as usize + 1;
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::failing_first(budget),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);
        dispatcher
            .dispatch(vec![consumed(p_old, partition, 0)])
            .await;

        dispatcher
            .dispatch_merges(vec![
                consumed_merge(p_old, p_new, partition, 10),
                consumed_merge(p_old, p_new, partition, 10),
            ])
            .await;
        let _tracker = dispatcher.shutdown().await;

        let transfers = transfer_sink.transfers();
        assert_eq!(transfers.len(), 1, "exactly one copy was ever acked");
        assert_eq!(
            (transfers[0].source_partition, transfers[0].source_offset),
            (partition, 10),
            "the re-produced copy carries the original source coordinates (the apply-side dedup key)",
        );
        assert!(
            store
                .get_pending_transfer(&pending_key_of(partition, p_old))
                .unwrap()
                .is_none(),
            "the outbox was cleared after the redelivered produce acked",
        );
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&11),
        );
    }

    #[tokio::test(start_paused = true)]
    async fn route_redrive_recovers_an_inline_exhausted_transfer_within_the_tenure() {
        let budget = TransferRetryPolicy::default().max_retries as usize + 1;
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::failing_first(budget),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(p_old, partition, 0)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 10)])
            .await;
        dispatcher.route_redrive().await;
        let _tracker = dispatcher.shutdown().await;

        let transfers = transfer_sink.transfers();
        assert_eq!(transfers.len(), 1, "the redrive's single attempt acked");
        assert_eq!(
            (transfers[0].source_partition, transfers[0].source_offset),
            (partition, 10),
            "the redriven copy carries the original source coordinates",
        );
        assert!(
            store
                .get_pending_transfer(&pending_key_of(partition, p_old))
                .unwrap()
                .is_none(),
            "the redrive cleared the outbox after the ack",
        );
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&11),
            "the stored merge-message coords marked past the exhausted merge",
        );
    }

    #[tokio::test]
    async fn route_redrive_duplicate_of_an_already_acked_transfer_applies_once() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            catalog.clone(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);
        dispatcher
            .dispatch(vec![consumed(p_old, partition, 0)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 5)])
            .await;
        let _tracker = dispatcher.shutdown().await;
        let copy_1 = {
            let transfers = transfer_sink.transfers();
            assert_eq!(transfers.len(), 1, "copy 1 acked inline");
            transfers.into_iter().next().unwrap()
        };

        // Simulate a crash between produce ack and outbox clear.
        store
            .write_batch(|batch| {
                batch.put_pending_transfer(
                    &pending_key_of(partition, p_old),
                    &PendingTransfer {
                        transfer: copy_1,
                        merge_msg_partition: partition,
                        merge_msg_offset: 5,
                    }
                    .encode(),
                )
            })
            .unwrap();

        let dispatcher_b = EventDispatcher::new(
            PartitionRouter::new(64),
            Arc::new(OffsetTracker::new()),
            test_handle(&store),
            catalog.clone(),
            Arc::new(CaptureSink::new()),
            merge.clone(),
        );
        dispatcher_b.assign_partition(partition);
        dispatcher_b
            .dispatch(vec![consumed(person(2), partition, 1)])
            .await;
        dispatcher_b.route_redrive().await;
        let _tracker_b = dispatcher_b.shutdown().await;

        let copies = transfer_sink.transfers();
        assert_eq!(copies.len(), 2, "the redrive minted the duplicate copy");
        assert_eq!(copies[0], copies[1], "byte-identical source coordinates");
        assert!(store
            .get_pending_transfer(&pending_key_of(partition, p_old))
            .unwrap()
            .is_none());

        let target = partition_of(TeamId(TEAM), &p_new, COHORT_PARTITION_COUNT) as i32;
        let apply_sink = CaptureSink::new();
        let dispatcher_c = EventDispatcher::new(
            PartitionRouter::new(64),
            Arc::new(OffsetTracker::new()),
            test_handle(&store),
            catalog,
            Arc::new(apply_sink.clone()),
            merge.clone(),
        );
        dispatcher_c.assign_partition(target);
        dispatcher_c
            .dispatch_transfers(vec![
                ConsumedTransfer {
                    transfer: copies[0].clone(),
                    partition: target,
                    offset: 0,
                },
                ConsumedTransfer {
                    transfer: copies[1].clone(),
                    partition: target,
                    offset: 1,
                },
            ])
            .await;
        let _tracker_c = dispatcher_c.shutdown().await;

        let changes = apply_sink.changes();
        assert_eq!(changes.len(), 1, "exactly one membership change end to end");
        assert_eq!(changes[0].person_id, p_new.to_string());
        assert_eq!(changes[0].status, MembershipStatus::Entered);
        assert_eq!(
            merge.transfer_tracker.committable_offsets().get(&target),
            Some(&2),
            "both copies settled (applied + AlreadyApplied)",
        );
    }

    #[tokio::test]
    async fn route_merge_gc_is_benign_with_no_workers_or_unowned_partitions() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.route_merge_gc(1_000, 500).await;
        dispatcher.assign_partition(0);
        dispatcher.route_merge_gc(1_000, 500).await;
        assert_eq!(dispatcher.workers.len(), 0, "GC never spawns a worker");
        assert_eq!(
            dispatcher.router.partition_count(),
            0,
            "no channel registered"
        );
    }

    #[tokio::test]
    async fn route_merge_gc_round_trip_deletes_expired_markers_and_keeps_fresh_ones() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let dispatcher = dispatcher_with(&store, catalog);

        let partition = 3i32;
        let partition_id = partition as u16;
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(person(1), partition, 0)])
            .await;

        let marker_cutoff = 10_000;
        let tombstone_cutoff = 5_000;
        let expired = TombstoneKey {
            partition_id,
            team_id: TEAM as u64,
            person: person(10),
        };
        let fresh = TombstoneKey {
            partition_id,
            team_id: TEAM as u64,
            person: person(11),
        };
        store
            .write_batch(|batch| {
                batch.put_tombstone(
                    &expired,
                    &Tombstone {
                        new_person: person(99),
                        merged_at_ms: tombstone_cutoff - 1,
                    }
                    .encode(),
                );
                batch.put_tombstone(
                    &fresh,
                    &Tombstone {
                        new_person: person(99),
                        merged_at_ms: tombstone_cutoff + 1,
                    }
                    .encode(),
                );
            })
            .unwrap();

        dispatcher
            .route_merge_gc(marker_cutoff, tombstone_cutoff)
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert!(
            store.get_tombstone(&expired).unwrap().is_none(),
            "the expired tombstone was GC'd end to end",
        );
        assert!(
            store.get_tombstone(&fresh).unwrap().is_some(),
            "the in-retention tombstone survived",
        );
    }

    #[tokio::test]
    async fn mixed_batch_marks_each_tracker_and_a_membership_produce_failure_holds_none() {
        let (_dir, store) = temp_store();
        let (dispatcher, sink, _transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::failing_first(1),
            CaptureTransferSink::new(),
        );

        let p_new = person_hashing_to(3, 200);
        let partition = partition_of(TeamId(TEAM), &p_new, COHORT_PARTITION_COUNT) as i32;
        let p_old = person(60); // no state → a fast-path no-op drain
        dispatcher.assign_partition(partition);

        dispatcher
            .dispatch(vec![consumed(person(1), partition, 10)])
            .await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 4)])
            .await;
        dispatcher
            .dispatch_transfers(vec![ConsumedTransfer {
                transfer: transfer_with(person(70), person(71), (9, 99), vec![]),
                partition,
                offset: 6,
            }])
            .await;
        let tracker = dispatcher.shutdown().await;

        assert!(
            sink.changes().is_empty(),
            "the failed produce recorded nothing"
        );
        assert_eq!(
            tracker.committable_offsets().get(&partition),
            Some(&11),
            "live output is at most once, so the failed produce does not hold the events offset",
        );
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&5),
            "the merge offset marks independently",
        );
        assert_eq!(
            merge.transfer_tracker.committable_offsets().get(&partition),
            Some(&7),
            "the transfer offset marks independently",
        );
    }

    #[tokio::test]
    async fn unowned_merge_and_transfer_are_dropped_without_raising_any_ceiling() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        dispatcher
            .dispatch_merges(vec![consumed_merge(person(1), person(2), 0, 5)])
            .await;
        dispatcher
            .dispatch_transfers(vec![ConsumedTransfer {
                transfer: transfer_with(person(1), person(2), (0, 5), vec![]),
                partition: 0,
                offset: 9,
            }])
            .await;

        assert_eq!(dispatcher.workers.len(), 0, "no worker spawned");
        assert_eq!(merge.merge_tracker.partition_count(), 0);
        assert_eq!(merge.transfer_tracker.partition_count(), 0);
        assert!(transfer_sink.transfers().is_empty());
        let _tracker = dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn draining_gate_rejects_all_dispatch_after_shutdown_without_hanging() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, _transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        dispatcher.assign_partition(0);
        dispatcher.dispatch(vec![consumed(person(1), 0, 10)]).await;
        let tracker = dispatcher.shutdown().await;
        assert_eq!(tracker.committable_offsets().get(&0), Some(&11));

        dispatcher.dispatch(vec![consumed(person(2), 0, 11)]).await;
        dispatcher
            .dispatch_merges(vec![consumed_merge(person(3), person(4), 0, 5)])
            .await;
        dispatcher.route_sweep(123).await;
        dispatcher.route_redrive().await;
        dispatcher.ensure_worker(0);

        assert_eq!(
            dispatcher.workers.len(),
            0,
            "no worker registered post-shutdown"
        );
        assert_eq!(
            dispatcher.router.partition_count(),
            0,
            "no sender registered"
        );
        assert_eq!(
            tracker.committable_offsets().get(&0),
            Some(&11),
            "the late event neither marked nor raised a ceiling",
        );
        assert_eq!(merge.merge_tracker.partition_count(), 0);
        let _tracker = dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn post_shutdown_registration_attempt_is_refused_by_the_closed_router() {
        let (_dir, store) = temp_store();
        let dispatcher = dispatcher_with(&store, behavioral_catalog());

        dispatcher.assign_partition(0);
        dispatcher.dispatch(vec![consumed(person(1), 0, 10)]).await;
        let tracker = dispatcher.shutdown().await;

        assert!(
            dispatcher.router.add_partition(0).is_none(),
            "the closed router refuses a late registration",
        );
        assert_eq!(
            dispatcher.router.partition_count(),
            0,
            "no sender was inserted for the refused registration",
        );
        assert_eq!(dispatcher.workers.len(), 0, "no worker to strand");
        assert_eq!(
            tracker.committable_offsets().get(&0),
            Some(&11),
            "the pre-shutdown work still committed",
        );
        let _tracker = dispatcher.shutdown().await;
    }

    #[tokio::test]
    async fn team_without_cohorts_merge_still_drains_tombstone_and_marker_and_marks() {
        let (_dir, store) = temp_store();
        let (dispatcher, sink, transfer_sink, merge) = dispatcher_full(
            &store,
            team_without_cohorts_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_new = person(2);
        let partition = partition_of(TeamId(TEAM), &p_new, COHORT_PARTITION_COUNT) as i32;
        let p_old = person(1);
        let lsk = LeafStateKey([0xAB; 16]);
        let p_old_key = BehavioralKey::new(partition as u16, TEAM as u64, p_old, lsk);
        store
            .write_batch(|batch| {
                batch.put::<Behavioral>(&p_old_key, &behavioral_match_record().encode());
            })
            .unwrap();

        dispatcher.assign_partition(partition);
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 9)])
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert!(
            store
                .get_tombstone(&TombstoneKey {
                    partition_id: partition as u16,
                    team_id: TEAM as u64,
                    person: p_old,
                })
                .unwrap()
                .is_some(),
            "the tombstone is load-bearing even with no cohorts",
        );
        assert!(store
            .get_merge_drain_applied(&MergeDrainKey {
                partition_id: partition as u16,
                team_id: TEAM as u64,
                old_person: p_old,
                merge_msg_partition: partition,
                merge_msg_offset: 9,
            })
            .unwrap()
            .is_some(),);
        assert!(
            store.get_behavioral(&p_old_key).unwrap().is_none(),
            "P_old's state was deleted",
        );
        assert!(sink.changes().is_empty(), "drifted leaves emit nothing");
        assert!(transfer_sink.transfers().is_empty());
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&10),
            "a team without cohorts never holds the offset",
        );
    }

    #[tokio::test]
    async fn team_without_cohorts_transfer_applies_dropping_leaves_as_drift_and_marks() {
        let (_dir, store) = temp_store();
        let (dispatcher, sink, _transfer_sink, merge) = dispatcher_full(
            &store,
            team_without_cohorts_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_old = person(1);
        let p_new = person(2);
        let lsk = LeafStateKey([0xAB; 16]);
        dispatcher.assign_partition(0);
        dispatcher
            .dispatch_transfers(vec![ConsumedTransfer {
                transfer: transfer_with(
                    p_old,
                    p_new,
                    (3, 7),
                    vec![TransferLeaf::new(lsk, behavioral_match_record())],
                ),
                partition: 0,
                offset: 4,
            }])
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert!(
            store
                .get_merge_applied(&MergeAppliedKey {
                    partition_id: 0,
                    team_id: TEAM as u64,
                    new_person: p_new,
                    source_partition: 3,
                    source_offset: 7,
                })
                .unwrap()
                .is_some(),
            "the apply marker still commits so the transfer is settled, not wedged",
        );
        assert!(
            store
                .get_behavioral(&BehavioralKey::new(0, TEAM as u64, p_new, lsk))
                .unwrap()
                .is_none(),
            "the drifted leaf wrote no state",
        );
        assert!(sink.changes().is_empty());
        assert_eq!(
            merge.transfer_tracker.committable_offsets().get(&0),
            Some(&5),
        );
    }

    #[tokio::test]
    async fn empty_transfer_is_skipped_not_produced_or_staged_and_marks() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_old = person(1);
        let partition = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as i32;
        let p_new = person_hashing_away_from(partition as u16, 100);
        dispatcher.assign_partition(partition);
        dispatcher
            .dispatch_merges(vec![consumed_merge(p_old, p_new, partition, 8)])
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert!(transfer_sink.transfers().is_empty(), "nothing to ferry");
        assert!(
            store
                .get_pending_transfer(&pending_key_of(partition, p_old))
                .unwrap()
                .is_none(),
            "nothing staged, so nothing to clear",
        );
        assert!(
            store
                .get_tombstone(&TombstoneKey {
                    partition_id: partition as u16,
                    team_id: TEAM as u64,
                    person: p_old,
                })
                .unwrap()
                .is_some(),
            "the tombstone still commits",
        );
        assert_eq!(
            merge.merge_tracker.committable_offsets().get(&partition),
            Some(&9),
        );
    }

    #[tokio::test]
    async fn apply_marks_the_transfer_offset_even_when_the_membership_produce_fails() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let lsk = behavioral_lsk(&catalog);
        let (dispatcher, sink, _transfer_sink, merge) = dispatcher_full(
            &store,
            catalog,
            CaptureSink::failing_first(1),
            CaptureTransferSink::new(),
        );

        let p_new = person(2);
        dispatcher.assign_partition(0);
        dispatcher
            .dispatch_transfers(vec![ConsumedTransfer {
                transfer: transfer_with(
                    person(1),
                    p_new,
                    (3, 7),
                    vec![TransferLeaf::new(lsk, behavioral_match_record())],
                ),
                partition: 0,
                offset: 6,
            }])
            .await;
        let _tracker = dispatcher.shutdown().await;

        assert!(
            sink.changes().is_empty(),
            "the failed flush recorded nothing"
        );
        assert!(
            matches!(
                behavioral_state(&store, 0, p_new, lsk),
                Some(Stage1State::BehavioralSingle {
                    has_match: true,
                    ..
                })
            ),
            "the merged state committed",
        );
        assert_eq!(
            merge.transfer_tracker.committable_offsets().get(&0),
            Some(&7),
        );
    }

    #[tokio::test]
    async fn dispatch_transfers_applies_emits_membership_and_marks() {
        let (_dir, store) = temp_store();
        let catalog = behavioral_catalog();
        let lsk = behavioral_lsk(&catalog);
        let (dispatcher, sink, _transfer_sink, merge) = dispatcher_full(
            &store,
            catalog,
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let p_new = person(2);
        dispatcher.assign_partition(0);
        dispatcher
            .dispatch_transfers(vec![ConsumedTransfer {
                transfer: transfer_with(
                    person(1),
                    p_new,
                    (3, 7),
                    vec![TransferLeaf::new(lsk, behavioral_match_record())],
                ),
                partition: 0,
                offset: 6,
            }])
            .await;
        let _tracker = dispatcher.shutdown().await;

        let changes = sink.changes();
        assert_eq!(changes.len(), 1, "P_new entered the single-leaf cohort");
        assert_eq!(changes[0].person_id, p_new.to_string());
        assert_eq!(changes[0].cohort_id, 1);
        assert_eq!(changes[0].status, MembershipStatus::Entered);
        assert_eq!(
            merge.transfer_tracker.committable_offsets().get(&0),
            Some(&7),
        );
    }

    fn stage_pending(
        store: &CohortStore,
        partition: i32,
        p_old: Uuid,
        p_new: Uuid,
        merge_offset: i64,
    ) -> (PendingTransferKey, MergeStateTransfer) {
        let transfer = transfer_with(p_old, p_new, (partition, merge_offset), vec![]);
        let key = pending_key_of(partition, p_old);
        let pending = PendingTransfer {
            transfer: transfer.clone(),
            merge_msg_partition: partition,
            merge_msg_offset: merge_offset,
        };
        store
            .write_batch(|b| b.put_pending_transfer(&key, &pending.encode()))
            .unwrap();
        (key, transfer)
    }

    #[tokio::test]
    async fn eager_boot_redrive_re_produces_a_restored_pending_transfer_without_a_worker() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let partition = 3;
        let (key, transfer) = stage_pending(&store, partition, person(1), person(2), 41);
        dispatcher.assign_partition(partition);

        // No dispatch yet: dispatched_offset == 0, so mark_processed would clamp and trip
        // CappedAheadOfDispatch — the eager redrive correctly skips it.
        let owned: HashSet<i32> = [partition].into_iter().collect();
        dispatcher
            .eager_redrive_pending_transfers_on_boot(&owned)
            .await;

        assert_eq!(
            transfer_sink.transfers(),
            vec![transfer],
            "the stranded transfer was re-produced with no worker spawned",
        );
        assert!(
            store.get_pending_transfer(&key).unwrap().is_none(),
            "the outbox entry was cleared after the ack",
        );
        assert!(
            merge.merge_tracker.committable_offsets().is_empty(),
            "the eager redrive marks no offset (would clamp to dispatched_offset==0 and trip F1)",
        );
        assert_eq!(
            merge.merge_tracker.partition_count(),
            0,
            "no tracker entry was created — the merge offset was already committed-past",
        );
        assert!(
            !dispatcher.workers.contains_key(&partition),
            "the eager redrive does not spawn a worker",
        );
    }

    #[tokio::test]
    async fn eager_boot_redrive_is_a_noop_for_an_empty_outbox() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        dispatcher.assign_partition(0);
        dispatcher.assign_partition(1);
        let owned: HashSet<i32> = [0, 1].into_iter().collect();
        dispatcher
            .eager_redrive_pending_transfers_on_boot(&owned)
            .await;

        assert!(
            transfer_sink.transfers().is_empty(),
            "an empty outbox produces nothing",
        );
        assert_eq!(
            merge.merge_tracker.partition_count(),
            0,
            "an empty outbox marks no offset",
        );
    }

    #[tokio::test]
    async fn eager_boot_redrive_only_touches_owned_partitions() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, _merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let owned_partition = 3;
        let unowned_partition = 7;
        let (owned_key, owned_transfer) =
            stage_pending(&store, owned_partition, person(1), person(2), 41);
        let (unowned_key, _unowned_transfer) =
            stage_pending(&store, unowned_partition, person(3), person(4), 51);

        dispatcher.assign_partition(owned_partition);
        let owned: HashSet<i32> = [owned_partition].into_iter().collect();
        dispatcher
            .eager_redrive_pending_transfers_on_boot(&owned)
            .await;

        assert_eq!(
            transfer_sink.transfers(),
            vec![owned_transfer],
            "only the owned partition's transfer was re-produced",
        );
        assert!(
            store.get_pending_transfer(&owned_key).unwrap().is_none(),
            "the owned partition's outbox entry was cleared",
        );
        assert!(
            store.get_pending_transfer(&unowned_key).unwrap().is_some(),
            "the unowned partition's outbox entry was left untouched",
        );
    }

    #[tokio::test]
    async fn eager_boot_redrive_paginates_until_the_whole_outbox_is_drained() {
        let (_dir, store) = temp_store();
        let (dispatcher, _sink, transfer_sink, _merge) = dispatcher_full(
            &store,
            behavioral_catalog(),
            CaptureSink::new(),
            CaptureTransferSink::new(),
        );

        let partition = 3;
        let store_partition = partition_to_store_id(partition).unwrap();
        // Seed five entries so a page size of two spans three pages (2 + 2 + 1).
        let mut expected = Vec::new();
        for n in 1..=5u128 {
            let (_key, transfer) =
                stage_pending(&store, partition, person(n), person(100 + n), 40 + n as i64);
            expected.push(transfer);
        }

        let recovered = dispatcher
            .eager_redrive_partition(partition, store_partition, 2)
            .await;

        assert_eq!(
            recovered, 5,
            "every staged entry was re-produced across pages"
        );
        // Production follows key order across pages; sort both sides on the stable old-person key so
        // the assertion does not depend on the seeding order.
        let mut produced = transfer_sink.transfers();
        produced.sort_by_key(|t| t.old_person_uuid);
        expected.sort_by_key(|t| t.old_person_uuid);
        assert_eq!(produced, expected);
        assert!(
            store
                .scan_pending_transfers(store_partition, None, usize::MAX)
                .unwrap()
                .is_empty(),
            "the outbox is fully drained — no entry stranded past the first page",
        );
    }
}