Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

94.7 KB
Raw Blame
//! In-memory reconcile work admitted by the seed consumer.
//!
//! The queue belongs to one partition worker. Its deferred offsets deliberately die with the
//! worker without completing; partition teardown then forgets the tracker tenure so Kafka replays
//! every unfinished reconcile from the beginning.

use std::collections::VecDeque;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::Instant;

use metrics::{counter, gauge, histogram};
use tracing::{debug, info, warn};

use cohort_core::clickhouse_timestamp_to_millis;
use cohort_core::filters::{CohortId, TeamId};
#[cfg(test)]
use cohort_core::seed::RunId;
use cohort_core::seed::{CatalogMiss, ReconcileScope, ReconcileTile, ScopeKind, WithheldReason};
use cohort_core::CohortEligibility;

use crate::filters::manager::{CatalogHandle, CatalogView, RefreshSeq};
use crate::filters::reverse_index::TeamFilters;
use crate::observability::metrics::{
    COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH, RECONCILE_BITS_FIXED_TOTAL,
    RECONCILE_JOBS_COMPLETED_TOTAL, RECONCILE_JOBS_DEFERRED_TOTAL, RECONCILE_JOBS_WITHHELD_TOTAL,
    RECONCILE_MARKERS_EMITTED_TOTAL, RECONCILE_MARKER_PRODUCE_ERRORS,
    RECONCILE_PAGE_DURATION_SECONDS, RECONCILE_QUEUE_DEPTH, RECONCILE_ROWS_EMITTED_TOTAL,
    RECONCILE_ROWS_SCANNED_TOTAL,
};
use crate::partitions::offset_tracker::{DeferredOffset, MarkOutcome, OffsetTracker};
use crate::producer::{
    ChangeOrigin, CohortMembershipChange, MembershipSink, NoopReconcileMarkerSink,
    ReconcileCompleteMarker, ReconcileMarker, ReconcileMarkerSink, ReconcileWithheldMarker,
};
use crate::stage2::Stage2State;
use crate::store::{
    ReadLane, SliceCoverage, Stage2CohortPrefix, Stage2DirtyKey, Stage2Key, StagedBatch,
    StoreHandle,
};
use crate::workers::merge_path::MergeWorkerDeps;
use crate::workers::reconcile_page::read_page;
#[cfg(test)]
use crate::workers::reconcile_page::MAX_ROWS;
use crate::workers::worker::{
    count_by_status, first_cascades, produce_cascades, produce_membership,
};

/// Emit a `warn!` carrying the reconcile job identity (`team_id`, `cohort_id`, `run_id`) read from
/// `$tile`, so the drain warnings share one definition of those fields. Extra fields and the message
/// follow exactly as in a bare `warn!`: `warn_job!(tile, partition_id, error = %error, "…")`.
macro_rules! warn_job {
    ($tile:expr, $($rest:tt)*) => {
        warn!(
            team_id = $tile.team_id().0,
            cohort_id = $tile.cohort_id().0,
            run_id = %$tile.run_id().0,
            $($rest)*
        )
    };
}

/// Default number of Stage 2 rows one reconcile drain tick may scan.
pub const DEFAULT_RECONCILE_SCAN_PAGE: usize = 256;

/// Pod-wide count of admitted jobs still owned by partition queues.
#[derive(Debug, Default)]
pub struct ReconcileBacklog(AtomicI64);

impl ReconcileBacklog {
    pub fn len(&self) -> i64 {
        self.0.load(Ordering::Relaxed)
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    fn add(&self) {
        self.0.fetch_add(1, Ordering::Relaxed);
    }

    fn done(&self, count: usize) {
        let count = i64::try_from(count).expect("an in-memory queue cannot exceed i64::MAX jobs");
        let updated = self
            .0
            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
                current.checked_sub(count).filter(|next| *next >= 0)
            });
        debug_assert!(updated.is_ok(), "reconcile backlog ownership underflow");
    }
}

/// Runtime dependencies shared by every partition's reconcile queue.
#[derive(Clone)]
pub struct ReconcileDeps {
    pub enabled: bool,
    pub scan_page: usize,
    pub backlog: Arc<ReconcileBacklog>,
    /// Sink for `cohort_reconcile_markers`. With the reconcile gate off this is
    /// [`NoopReconcileMarkerSink`], which fails every produce rather than acking a marker the seeder
    /// would then wait for.
    pub marker_sink: Arc<dyn ReconcileMarkerSink>,
}

impl Default for ReconcileDeps {
    fn default() -> Self {
        Self {
            enabled: false,
            scan_page: DEFAULT_RECONCILE_SCAN_PAGE,
            backlog: Arc::new(ReconcileBacklog::default()),
            marker_sink: Arc::new(NoopReconcileMarkerSink),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
enum ScanPhase {
    Scanning { cursor: Option<Vec<u8>> },
    DrainingDirty { cursor: Option<Vec<u8>> },
    MarkerReady,
}

#[derive(Debug)]
struct ReconcileJob {
    tile: ReconcileTile,
    offset: DeferredOffset,
    /// The newest catalog refresh that had begun when the job was admitted. Only a snapshot from a
    /// later refresh can discard it.
    admitted_after: RefreshSeq,
    /// Set the first time the job waits for that refresh, so the wait counts and logs once.
    deferral_counted: bool,
    dirty_tracking: Option<crate::store::Stage2DirtyTrackingGuard>,
    phase: ScanPhase,
    rows_scanned: u64,
    bits_fixed: u64,
}

pub(crate) enum SupersedeOutcome {
    NoQueuedJob,
    Replaced(DeferredOffset),
    RetainedNewerOrEqual,
}

/// FIFO reconcile work for one partition worker.
pub(crate) struct ReconcileQueue {
    jobs: VecDeque<ReconcileJob>,
    backlog: Arc<ReconcileBacklog>,
    partition_label: Arc<str>,
    handle: StoreHandle,
    /// `enqueue` records the newest refresh this handle has begun on each job, and the drain compares
    /// it with the refresh this handle last published.
    catalog: Arc<CatalogHandle>,
    /// Fixed for the worker's life: a record changes only when its slice is deleted.
    coverage: SliceCoverage,
}

impl ReconcileQueue {
    pub(crate) fn new(
        partition_id: u16,
        backlog: Arc<ReconcileBacklog>,
        handle: StoreHandle,
        catalog: Arc<CatalogHandle>,
        coverage: SliceCoverage,
    ) -> Self {
        let queue = Self {
            jobs: VecDeque::new(),
            backlog,
            partition_label: Arc::from(partition_id.to_string()),
            handle,
            catalog,
            coverage,
        };
        queue.record_depth();
        queue
    }

    pub(crate) fn enqueue(&mut self, tile: ReconcileTile, offset: DeferredOffset) {
        self.jobs.push_back(ReconcileJob {
            tile,
            offset,
            admitted_after: self.catalog.last_begun(),
            deferral_counted: false,
            dirty_tracking: None,
            phase: ScanPhase::Scanning { cursor: None },
            rows_scanned: 0,
            bits_fixed: 0,
        });
        self.backlog.add();
        self.record_depth();
    }

    /// Start dirty capture only for the queue head, immediately before its first scan. Mutations
    /// while a job waits behind another snapshot are covered by its future full scan and need no
    /// per-person metadata.
    fn activate_front_tracking(&mut self, prefix: Stage2CohortPrefix) {
        if self
            .jobs
            .front()
            .is_some_and(|job| job.dirty_tracking.is_none())
        {
            let tracking = self.handle.track_stage2_dirty(prefix);
            self.jobs
                .front_mut()
                .expect("the queue head was present before acquiring its tracking lease")
                .dirty_tracking = Some(tracking);
        }
    }

    /// Replace a queued job only when the incoming Kafka offset is newer. A follower rewind may
    /// replay an older job while a newer run is still queued; that replay must never evict the newer
    /// snapshot.
    ///
    /// Scoped by kind as well as cohort: a mixed cohort can be reconciled by a behavioral and a
    /// person-property run at once, and those two snapshots answer different fences — evicting one
    /// for the other would leave its run permanently short a marker.
    pub(crate) fn supersede_if_newer(
        &mut self,
        team_id: TeamId,
        cohort_id: CohortId,
        scope: ScopeKind,
        incoming_offset: i64,
    ) -> SupersedeOutcome {
        let Some(position) = self.jobs.iter().position(|job| {
            job.tile.team_id() == team_id
                && job.tile.cohort_id() == cohort_id
                && job.tile.scope().kind() == scope
        }) else {
            return SupersedeOutcome::NoQueuedJob;
        };
        if incoming_offset <= self.jobs[position].offset.offset() {
            return SupersedeOutcome::RetainedNewerOrEqual;
        }
        let job = self
            .jobs
            .remove(position)
            .expect("position came from the same queue");
        self.backlog.done(1);
        self.record_depth();
        SupersedeOutcome::Replaced(job.offset)
    }

    fn front(&self) -> Option<&ReconcileJob> {
        self.jobs.front()
    }

    fn front_mut(&mut self) -> Option<&mut ReconcileJob> {
        self.jobs.front_mut()
    }

    fn finish_front(&mut self) -> Option<ReconcileJob> {
        let job = self.jobs.pop_front()?;
        self.backlog.done(1);
        self.record_depth();
        Some(job)
    }

    fn record_depth(&self) {
        gauge!(RECONCILE_QUEUE_DEPTH, "partition" => self.partition_label.clone())
            .set(self.jobs.len() as f64);
    }

    #[cfg(test)]
    pub(crate) fn len(&self) -> usize {
        self.jobs.len()
    }

    #[cfg(test)]
    pub(crate) fn front_run_id(&self) -> Option<RunId> {
        self.jobs.front().map(|job| job.tile.run_id())
    }
}

impl Drop for ReconcileQueue {
    fn drop(&mut self) {
        self.backlog.done(self.jobs.len());
        gauge!(RECONCILE_QUEUE_DEPTH, "partition" => self.partition_label.clone()).set(0.0);
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReconcileRetryReason {
    CatalogNotLoaded,
    /// The snapshot predates the job, so the miss may be a write it has not read yet.
    AwaitingRefresh(CatalogMiss),
}

#[derive(Debug)]
enum ReconcileGuard {
    /// Carries the team filters and eligibility the guard found, so the drain does not look them up
    /// again.
    Proceed {
        filters: Arc<TeamFilters>,
        eligibility: CohortEligibility,
    },
    Retry(ReconcileRetryReason),
    Withhold(WithheldReason),
}

/// The catalog's answer alone, with no view of how fresh the catalog is.
fn catalog_verdict(
    filters: Option<&Arc<TeamFilters>>,
    tile: &ReconcileTile,
) -> Result<(Arc<TeamFilters>, CohortEligibility), CatalogMiss> {
    let Some(filters) = filters else {
        return Err(CatalogMiss::TeamAbsent);
    };
    if !filters.cohorts.contains_key(&tile.cohort_id()) {
        return Err(CatalogMiss::CohortAbsent);
    }
    let Some(&eligibility) = filters.eligibility.get(&tile.cohort_id()) else {
        return Err(CatalogMiss::CohortAbsent);
    };
    if !eligibility.registers_membership() {
        return Err(CatalogMiss::NotEmitting);
    }
    // The tile's scope picks the guard map, so a person hash is never checked against a behavioral
    // one. An absent entry means the cohort has no leaves of that kind (the loader omits the column
    // when it is empty) or the run was superseded; either way this is the intended fail-closed skip,
    // observable via the `hash_unknown` withheld marker.
    let matches = match tile.scope() {
        ReconcileScope::Behavioral(pinned) => filters
            .behavioral_shape_hashes
            .get(&tile.cohort_id())
            .map(|current| current == pinned),
        ReconcileScope::PersonProperty(pinned) => filters
            .person_shape_hashes
            .get(&tile.cohort_id())
            .map(|current| current == pinned),
    };
    match matches {
        None => Err(CatalogMiss::HashUnknown),
        Some(false) => Err(CatalogMiss::HashMismatch),
        Some(true) => Ok((filters.clone(), eligibility)),
    }
}

/// The one place that decides a job's fate. Every catalog miss reads the catalog, so a miss
/// against a snapshot whose refresh began before the job was admitted may only mean that refresh
/// had not read the run's writes yet. Such a miss waits. After one later refresh it stands, so a
/// superseded run still fails closed.
///
/// Coverage is checked after the catalog and before any scan, so a partial slice emits no rows.
fn evaluate_guard(
    view: &CatalogView,
    admitted_after: RefreshSeq,
    tile: &ReconcileTile,
    coverage: SliceCoverage,
) -> ReconcileGuard {
    let (published, catalog) = match view {
        CatalogView::NotLoaded => {
            return ReconcileGuard::Retry(ReconcileRetryReason::CatalogNotLoaded)
        }
        CatalogView::Loaded { published, catalog } => (*published, catalog),
    };
    match catalog_verdict(catalog.team(tile.team_id()), tile) {
        Err(miss) if published > admitted_after => {
            ReconcileGuard::Withhold(WithheldReason::Catalog(miss))
        }
        Err(miss) => ReconcileGuard::Retry(ReconcileRetryReason::AwaitingRefresh(miss)),
        Ok((filters, eligibility)) => match coverage.admits(tile.boundary()) {
            Ok(()) => ReconcileGuard::Proceed {
                filters,
                eligibility,
            },
            Err(covered_since) => {
                ReconcileGuard::Withhold(WithheldReason::PartialCoverage { covered_since })
            }
        },
    }
}

/// Advance the queue head by at most one scan page or one marker produce per tick.
#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
pub(crate) async fn handle_reconcile_drain(
    partition_id: u16,
    handle: &StoreHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    queue: &mut ReconcileQueue,
    last_updated: &str,
) {
    loop {
        let Some(job) = queue.front() else {
            return;
        };
        let tile = job.tile.clone();
        let phase = job.phase.clone();
        let source_offset = job.offset.offset();

        let (filters, eligibility) = match evaluate_guard(
            &queue.catalog.view(),
            job.admitted_after,
            &tile,
            queue.coverage,
        ) {
            ReconcileGuard::Retry(ReconcileRetryReason::CatalogNotLoaded) => {
                debug!(
                    partition_id,
                    team_id = tile.team_id().0,
                    cohort_id = tile.cohort_id().0,
                    run_id = %tile.run_id().0,
                    "reconcile drain waiting for the first filter catalog load",
                );
                return;
            }
            ReconcileGuard::Retry(ReconcileRetryReason::AwaitingRefresh(reason)) => {
                let job = queue
                    .front_mut()
                    .expect("the guarded queue head is still present");
                if !job.deferral_counted {
                    job.deferral_counted = true;
                    counter!(
                        RECONCILE_JOBS_DEFERRED_TOTAL,
                        "reason" => reason.as_str(),
                        "kind" => tile.scope().kind().as_str(),
                    )
                    .increment(1);
                    info!(
                        partition_id,
                        team_id = tile.team_id().0,
                        cohort_id = tile.cohort_id().0,
                        run_id = %tile.run_id().0,
                        reason = reason.as_str(),
                        kind = tile.scope().kind().as_str(),
                        "reconcile job waits for a catalog refresh that began after it arrived",
                    );
                }
                return;
            }
            ReconcileGuard::Withhold(reason) => {
                drain_withheld(partition_id, merge, queue, reason).await;
                return;
            }
            ReconcileGuard::Proceed {
                filters,
                eligibility,
            } => (filters, eligibility),
        };

        // Only flipped bits of a full-tree cohort cascade to referrers; single-leaf fixes never do.
        let cascades_flips = eligibility.writes_cf_stage2();

        let prefix = Stage2CohortPrefix {
            partition_id,
            team_id: tile.team_id().0 as u64,
            cohort_id: tile.cohort_id().0 as u64,
        };
        queue.activate_front_tracking(prefix);

        let step = match phase {
            ScanPhase::Scanning { cursor } => {
                drain_scanning(
                    partition_id,
                    handle,
                    sink,
                    merge,
                    queue,
                    &tile,
                    &filters,
                    cascades_flips,
                    prefix,
                    source_offset,
                    cursor,
                    last_updated,
                )
                .await
            }
            ScanPhase::DrainingDirty { cursor } => {
                drain_dirty(
                    partition_id,
                    handle,
                    sink,
                    merge,
                    queue,
                    &tile,
                    &filters,
                    cascades_flips,
                    prefix,
                    source_offset,
                    cursor,
                    last_updated,
                )
                .await
            }
            ScanPhase::MarkerReady => {
                drain_marker(
                    partition_id,
                    handle,
                    merge,
                    queue,
                    &tile,
                    prefix,
                    last_updated,
                )
                .await
            }
        };
        match step {
            DrainStep::Reenter => continue,
            DrainStep::Yield => return,
        }
    }
}

/// Whether the drain loop should re-enter against the head's newly transitioned phase
/// ([`DrainStep::Reenter`]) or yield the tick because its work budget is spent ([`DrainStep::Yield`]).
enum DrainStep {
    Reenter,
    Yield,
}

/// Scan one Stage 2 page for the cohort and emit its current membership. An empty page hands off to
/// dirty verification; a settled non-empty page is the tick's whole budget.
#[allow(clippy::too_many_arguments)]
async fn drain_scanning(
    partition_id: u16,
    handle: &StoreHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    queue: &mut ReconcileQueue,
    tile: &ReconcileTile,
    filters: &Arc<TeamFilters>,
    cascades_flips: bool,
    prefix: Stage2CohortPrefix,
    source_offset: i64,
    cursor: Option<Vec<u8>>,
    last_updated: &str,
) -> DrainStep {
    let page = match handle
        .scan_stage2_cohort(prefix, cursor, merge.reconcile.scan_page)
        .await
    {
        Ok(page) => page,
        Err(error) => {
            warn_job!(
                tile,
                partition_id,
                error = %error,
                "reconcile Stage 2 scan failed; retrying this page on the next tick",
            );
            return DrainStep::Yield;
        }
    };
    if page.is_empty() {
        queue
            .front_mut()
            .expect("the scanned queue head is still present")
            .phase = ScanPhase::DrainingDirty { cursor: None };
        return DrainStep::Reenter;
    }

    let dirty_to_clear: Vec<Stage2DirtyKey> =
        page.iter().copied().map(Stage2DirtyKey::new).collect();
    let Some(progress) = settle_reconcile_page(
        partition_id,
        handle,
        sink,
        merge,
        tile,
        filters,
        cascades_flips,
        &page,
        &dirty_to_clear,
        source_offset,
        last_updated,
    )
    .await
    else {
        return DrainStep::Yield;
    };

    let short_page = page.len() < merge.reconcile.scan_page;
    let next_cursor = page
        .last()
        .expect("a non-empty page has a final key")
        .encode()
        .to_vec();
    let job = queue
        .front_mut()
        .expect("the advanced queue head is still present");
    job.rows_scanned = job.rows_scanned.saturating_add(progress.rows_scanned);
    job.bits_fixed = job.bits_fixed.saturating_add(progress.bits_fixed);
    job.phase = if short_page {
        ScanPhase::DrainingDirty { cursor: None }
    } else {
        ScanPhase::Scanning {
            cursor: Some(next_cursor),
        }
    };
    // One non-empty settlement page is the tick's work budget. Even a short final main page leaves
    // dirty verification for the next tick, so mutations behind the cursor cannot silently double the
    // configured per-tick ceiling.
    DrainStep::Yield
}

/// Re-settle rows mutated behind the main scan, verifying from the cohort prefix until the dirty set
/// is empty before advancing to the marker.
#[allow(clippy::too_many_arguments)]
async fn drain_dirty(
    partition_id: u16,
    handle: &StoreHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    queue: &mut ReconcileQueue,
    tile: &ReconcileTile,
    filters: &Arc<TeamFilters>,
    cascades_flips: bool,
    prefix: Stage2CohortPrefix,
    source_offset: i64,
    cursor: Option<Vec<u8>>,
    last_updated: &str,
) -> DrainStep {
    let page = match handle
        .scan_stage2_dirty(prefix, cursor.clone(), merge.reconcile.scan_page)
        .await
    {
        Ok(page) => page,
        Err(error) => {
            warn_job!(
                tile,
                partition_id,
                error = %error,
                "reconcile dirty scan failed; retrying this page on the next tick",
            );
            return DrainStep::Yield;
        }
    };
    if page.is_empty() {
        let job = queue
            .front_mut()
            .expect("the dirty-scanning queue head is still present");
        if cursor.is_some() {
            // Verify once more from the prefix so a marker inserted behind the cursor
            // between ticks cannot be missed.
            job.phase = ScanPhase::DrainingDirty { cursor: None };
        } else {
            job.phase = ScanPhase::MarkerReady;
        }
        return DrainStep::Reenter;
    }

    let stage2_keys: Vec<Stage2Key> = page.iter().map(|dirty| dirty.stage2_key()).collect();
    let current_rows = match handle
        .multi_get_stage2(stage2_keys.clone(), ReadLane::Maintenance)
        .await
    {
        Ok(rows) => rows,
        Err(error) => {
            warn_job!(
                tile,
                partition_id,
                error = %error,
                "reconcile dirty-row read failed; retrying this page on the next tick",
            );
            return DrainStep::Yield;
        }
    };
    let existing_keys: Vec<Stage2Key> = stage2_keys
        .into_iter()
        .zip(current_rows)
        .filter_map(|(key, value)| value.is_some().then_some(key))
        .collect();
    let Some(progress) = settle_reconcile_page(
        partition_id,
        handle,
        sink,
        merge,
        tile,
        filters,
        cascades_flips,
        &existing_keys,
        &page,
        source_offset,
        last_updated,
    )
    .await
    else {
        return DrainStep::Yield;
    };

    let short_page = page.len() < merge.reconcile.scan_page;
    let next_cursor = page
        .last()
        .expect("a non-empty dirty page has a final key")
        .encode()
        .to_vec();
    let job = queue
        .front_mut()
        .expect("the advanced dirty queue head is still present");
    job.rows_scanned = job.rows_scanned.saturating_add(progress.rows_scanned);
    job.bits_fixed = job.bits_fixed.saturating_add(progress.bits_fixed);
    job.phase = ScanPhase::DrainingDirty {
        cursor: Some(next_cursor.clone()),
    };

    // A full page does not prove exhaustion, so first look strictly after it. This is
    // verification only: if another page exists, the next tick settles it.
    if !short_page {
        match handle.scan_stage2_dirty(prefix, Some(next_cursor), 1).await {
            Ok(remaining) if !remaining.is_empty() => return DrainStep::Yield,
            Ok(_) => {}
            Err(error) => {
                warn_job!(
                    tile,
                    partition_id,
                    error = %error,
                    "reconcile dirty tail verification failed; retrying next tick",
                );
                return DrainStep::Yield;
            }
        }
    }

    // Verify once from the prefix to catch work inserted behind an earlier cursor. The
    // worker owns this partition serially, so an empty result immediately after the
    // settlement closes the hot-single-key case even when page size is one. Never
    // settle a second nonempty page in this tick.
    queue
        .front_mut()
        .expect("the verified dirty queue head is still present")
        .phase = ScanPhase::DrainingDirty { cursor: None };
    match handle.scan_stage2_dirty(prefix, None, 1).await {
        Ok(remaining) if !remaining.is_empty() => DrainStep::Yield,
        Ok(_) => {
            queue
                .front_mut()
                .expect("the verified dirty queue head is still present")
                .phase = ScanPhase::MarkerReady;
            DrainStep::Reenter
        }
        Err(error) => {
            warn_job!(
                tile,
                partition_id,
                error = %error,
                "reconcile dirty prefix verification failed; retrying next tick",
            );
            DrainStep::Yield
        }
    }
}

/// Withhold the consecutive withheld jobs at the head in one produce, releasing the acked prefix.
async fn drain_withheld(
    partition_id: u16,
    merge: &MergeWorkerDeps,
    queue: &mut ReconcileQueue,
    head: WithheldReason,
) {
    let view = queue.catalog.view();
    let coverage = queue.coverage;
    let reasons: Vec<WithheldReason> = std::iter::once(head)
        .chain(queue.jobs.iter().skip(1).map_while(|job| {
            match evaluate_guard(&view, job.admitted_after, &job.tile, coverage) {
                ReconcileGuard::Withhold(reason) => Some(reason),
                ReconcileGuard::Proceed { .. } | ReconcileGuard::Retry(_) => None,
            }
        }))
        // A tick produces at most one scan page of messages, as a scan tick does, so a burst spreads
        // over ticks instead of holding the live lane behind one unbounded produce.
        .take(merge.reconcile.scan_page)
        .collect();
    let markers = queue
        .jobs
        .iter()
        .zip(&reasons)
        .map(|(job, &reason)| {
            ReconcileMarker::Withheld(ReconcileWithheldMarker::new(
                job.tile.team_id(),
                job.tile.cohort_id(),
                partition_id,
                job.tile.run_id(),
                reason,
            ))
        })
        .collect();
    let acks = merge.reconcile.marker_sink.produce(markers).await;
    let acked = acks
        .iter()
        .take_while(|ack| ack.is_ok())
        .count()
        .min(reasons.len());

    for reason in &reasons[..acked] {
        let job = queue
            .finish_front()
            .expect("an acked withheld job is still queued");
        let kind = job.tile.scope().kind().as_str();
        counter!(RECONCILE_MARKERS_EMITTED_TOTAL, "kind" => kind, "marker" => "withheld")
            .increment(1);
        complete_offset(
            &merge.seed_tracker,
            partition_id,
            job.offset,
            "withheld reconcile",
        );
        counter!(
            RECONCILE_JOBS_WITHHELD_TOTAL,
            "reason" => reason.as_str(),
            "kind" => kind,
        )
        .increment(1);
        warn_job!(
            job.tile,
            partition_id,
            reason = reason.as_str(),
            covered_since_ms = reason.covered_since().map(|since| since.0),
            kind,
            "withholding the reconcile certificate",
        );
    }
    if acked < reasons.len() {
        let tile = &queue
            .front()
            .expect("the unacked withheld job is still queued")
            .tile;
        counter!(
            RECONCILE_MARKER_PRODUCE_ERRORS,
            "kind" => tile.scope().kind().as_str(),
            "marker" => "withheld",
        )
        .increment(1);
        warn_job!(
            tile,
            partition_id,
            unacked = reasons.len() - acked,
            "reconcile withheld-marker produce failed; deciding the jobs again on the next tick",
        );
    }
}

/// Emit the per-partition completion marker once the dirty set is drained. A failed marker produce
/// retries the marker only; work dirtied while the marker was pending is settled before the retry.
async fn drain_marker(
    partition_id: u16,
    handle: &StoreHandle,
    merge: &MergeWorkerDeps,
    queue: &mut ReconcileQueue,
    tile: &ReconcileTile,
    prefix: Stage2CohortPrefix,
    last_updated: &str,
) -> DrainStep {
    // A marker produce may have failed on a prior tick. Recheck the dirty prefix before
    // every retry so newly arrived work is settled first without rescanning the cohort.
    let pending_dirty = match handle.scan_stage2_dirty(prefix, None, 1).await {
        Ok(page) => page,
        Err(error) => {
            warn_job!(
                tile,
                partition_id,
                error = %error,
                "reconcile dirty verification failed before marker; retrying on the next tick",
            );
            return DrainStep::Yield;
        }
    };
    if !pending_dirty.is_empty() {
        queue
            .front_mut()
            .expect("the marker-ready queue head is still present")
            .phase = ScanPhase::DrainingDirty { cursor: None };
        return DrainStep::Reenter;
    }
    let marker = ReconcileMarker::Complete(ReconcileCompleteMarker::new(
        tile.team_id(),
        tile.cohort_id(),
        partition_id,
        tile.run_id(),
        last_updated.to_string(),
    ));
    let acks = merge.reconcile.marker_sink.produce(vec![marker]).await;
    let kind = tile.scope().kind().as_str();
    // Exactly one marker went in, so anything but a single successful ack is a produce failure.
    let failed_acks = acks.iter().filter(|ack| ack.is_err()).count();
    if acks.len() != 1 || failed_acks > 0 {
        counter!(RECONCILE_MARKER_PRODUCE_ERRORS, "kind" => kind, "marker" => "complete")
            .increment(1);
        warn_job!(
            tile,
            partition_id,
            ack_count = acks.len(),
            failed_acks,
            "reconcile completion-marker produce failed; rechecking dirty work before retry",
        );
        return DrainStep::Yield;
    }

    counter!(RECONCILE_MARKERS_EMITTED_TOTAL, "kind" => kind, "marker" => "complete").increment(1);
    let completed = queue
        .finish_front()
        .expect("the marker-producing queue head is still present");
    complete_offset(
        &merge.seed_tracker,
        partition_id,
        completed.offset,
        "completed reconcile",
    );
    counter!(RECONCILE_JOBS_COMPLETED_TOTAL, "kind" => kind).increment(1);
    info!(
        partition_id,
        team_id = tile.team_id().0,
        cohort_id = tile.cohort_id().0,
        run_id = %tile.run_id().0,
        rows_scanned = completed.rows_scanned,
        bits_fixed = completed.bits_fixed,
        "reconcile job completed",
    );
    DrainStep::Yield
}

struct PageProgress {
    rows_scanned: u64,
    bits_fixed: u64,
}

/// The ordered steps one settlement page takes. Doubles as the `stage` metric label, so the
/// duration histogram can never name a step the page does not have.
#[derive(Debug, Clone, Copy)]
enum PageStage {
    /// Every section of the page's read and evaluation, permit waits included.
    Recompute,
    MembershipProduce,
    CascadeProduce,
    Commit,
}

impl PageStage {
    fn as_str(self) -> &'static str {
        match self {
            Self::Recompute => "recompute",
            Self::MembershipProduce => "membership_produce",
            Self::CascadeProduce => "cascade_produce",
            Self::Commit => "commit",
        }
    }
}

/// Splits one page's wall clock into per-stage samples: each [`mark`](Self::mark) records the time
/// since the previous one. A page that fails records no sample for the step that failed, so the
/// histogram stays a picture of settled work.
struct PageClock {
    last: Instant,
}

impl PageClock {
    fn start() -> Self {
        Self {
            last: Instant::now(),
        }
    }

    fn mark(&mut self, stage: PageStage) {
        let now = Instant::now();
        histogram!(RECONCILE_PAGE_DURATION_SECONDS, "stage" => stage.as_str())
            .record(now.duration_since(self.last).as_secs_f64());
        self.last = now;
    }
}

/// Emit one current reconcile page, then atomically commit any corrected bits and clear the exact
/// dirty markers covered by that evaluation. Returning `None` leaves the queue phase unchanged so
/// the same page is retried on the next tick.
#[allow(clippy::too_many_arguments)]
async fn settle_reconcile_page(
    partition_id: u16,
    handle: &StoreHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    tile: &ReconcileTile,
    filters: &Arc<TeamFilters>,
    cascades_flips: bool,
    page: &[Stage2Key],
    dirty_to_clear: &[Stage2DirtyKey],
    source_offset: i64,
    last_updated: &str,
) -> Option<PageProgress> {
    let evaluated_at_ms = clickhouse_timestamp_to_millis(last_updated)
        .expect("worker-generated last_updated timestamps always parse");
    let mut clock = PageClock::start();
    let diffs = match read_page(handle, filters, tile.cohort_id(), page).await {
        Ok(diffs) => diffs,
        Err(error) => {
            warn_job!(
                tile,
                partition_id,
                rows = page.len(),
                error = %error,
                "reconcile page read failed; retrying this page on the next tick",
            );
            return None;
        }
    };
    clock.mark(PageStage::Recompute);
    debug_assert_eq!(
        diffs.len(),
        page.len(),
        "the page reader returns one diff per requested row",
    );

    let mut changes = Vec::with_capacity(diffs.len());
    let mut cascade_changes = Vec::new();
    let mut writes: Vec<(Stage2Key, Stage2State)> = Vec::new();
    let mut fixed_entered = 0u64;
    let mut fixed_left = 0u64;
    for diff in &diffs {
        let change = CohortMembershipChange {
            team_id: tile.team_id().0,
            cohort_id: tile.cohort_id().0,
            person_id: diff.stage2_key.person_id.to_string(),
            last_updated: last_updated.to_string(),
            status: diff.status(),
            origin: Some(ChangeOrigin::Reconcile),
            run_id: Some(tile.run_id()),
        };
        if diff.requires_write() {
            writes.push((
                diff.stage2_key,
                Stage2State {
                    in_cohort: diff.new_bit,
                    last_evaluated_at_ms: evaluated_at_ms,
                },
            ));
        }
        if diff.flipped() {
            if diff.new_bit {
                fixed_entered += 1;
            } else {
                fixed_left += 1;
            }
            if cascades_flips {
                cascade_changes.push(change.clone());
            }
        }
        changes.push(change);
    }

    let cascades = first_cascades(merge, &cascade_changes, source_offset);
    let (entered, left) = count_by_status(&changes);
    let membership_errors = if changes.is_empty() {
        0
    } else {
        produce_membership(sink, changes).await
    };
    if membership_errors > 0 {
        warn_job!(
            tile,
            partition_id,
            errors = membership_errors,
            "reconcile membership produce failed; retrying this page on the next tick",
        );
        return None;
    }
    clock.mark(PageStage::MembershipProduce);

    let cascade_errors = produce_cascades(merge, cascades).await;
    if cascade_errors > 0 {
        warn_job!(
            tile,
            partition_id,
            errors = cascade_errors,
            "reconcile cascade produce failed; retrying this page on the next tick",
        );
        return None;
    }
    clock.mark(PageStage::CascadeProduce);

    let mut staged = StagedBatch::default();
    for (key, state) in &writes {
        staged.put_stage2(key, &state.encode());
    }
    // These deletes are deliberately staged after the puts. A fix marks its row dirty through the
    // normal typed write path, then this page-clear consumes that marker in the same atomic batch.
    for dirty in dirty_to_clear {
        staged.delete_stage2_dirty(dirty);
    }
    if !staged.is_empty() {
        if let Err(error) = handle.commit(staged).await {
            warn_job!(
                tile,
                partition_id,
                error = %error,
                "reconcile Stage 2 settlement failed; retrying this page on the next tick",
            );
            return None;
        }
    }
    clock.mark(PageStage::Commit);
    // Count only durably-settled pages: membership produce, cascade produce, and commit have all
    // succeeded here. Any earlier failure returns `None` above and retries the whole page, so
    // incrementing here keeps a retry from double-counting.
    counter!(RECONCILE_ROWS_SCANNED_TOTAL).increment(diffs.len() as u64);
    if entered > 0 {
        counter!(RECONCILE_ROWS_EMITTED_TOTAL, "status" => "entered").increment(entered);
    }
    if left > 0 {
        counter!(RECONCILE_ROWS_EMITTED_TOTAL, "status" => "left").increment(left);
    }
    if fixed_entered > 0 {
        counter!(RECONCILE_BITS_FIXED_TOTAL, "direction" => "entered").increment(fixed_entered);
    }
    if fixed_left > 0 {
        counter!(RECONCILE_BITS_FIXED_TOTAL, "direction" => "left").increment(fixed_left);
    }

    Some(PageProgress {
        rows_scanned: diffs.len() as u64,
        bits_fixed: fixed_entered + fixed_left,
    })
}

fn complete_offset(
    tracker: &OffsetTracker,
    partition_id: u16,
    offset: DeferredOffset,
    operation: &'static str,
) {
    match tracker.complete_deferred(offset) {
        Some(MarkOutcome::WithinDispatch) => {}
        Some(MarkOutcome::CappedAheadOfDispatch) => {
            counter!(COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH).increment(1);
            warn!(
                partition_id,
                operation, "reconcile completion exceeded the seed dispatch ceiling",
            );
        }
        None => warn!(
            partition_id,
            operation, "reconcile completion belonged to an expired seed-tracker tenure",
        ),
    }
}

#[cfg(test)]
#[allow(clippy::disallowed_methods)]
mod tests {
    use std::sync::atomic::AtomicBool;

    use async_trait::async_trait;
    use chrono_tz::UTC;
    use common_kafka::kafka_producer::KafkaProduceError;
    use serde_json::{json, Value};
    use tempfile::TempDir;
    use uuid::Uuid;

    use cohort_core::seed::{BehavioralShapeHash, CoverageStartMs, PersonShapeHash, RunBoundaryMs};
    use cohort_core::{CohortEligibility, ExcludedReason, FilterCatalog, LeafStateKey};

    use crate::filters::tree::{CohortTree, FilterNode};
    use crate::filters::{BoolOp, TeamFiltersBuilder};
    use crate::partitions::offset_tracker::OffsetTracker;
    use crate::producer::{
        CaptureCascadeSink, CaptureReconcileMarkerSink, CaptureSink, MembershipStatus,
    };
    use crate::stage1::person_record::{MatchedSet, PersonRecord};
    use crate::stage1::state::{AppliedOffsets, Stage1State, StatefulRecord};
    use crate::stage2::state::Stage2Ownership;
    use crate::store::{
        Behavioral, BehavioralKey, CohortStore, OffloadConfig, OffloadMode, PersonRecordKey,
        PersonRecords, StoreConfig,
    };

    use super::*;

    const TEAM: i32 = 7;
    const COHORT: i32 = 1;
    const PARTITION: u16 = 0;
    const BEHAVIORAL_HASH: [u8; 16] = *b"0123456789abcdef";
    const PERSON_HASH: [u8; 16] = *b"fedcba9876543210";
    const SHAPE_HASH: &str = "shape-v1";
    const TS: &str = "2026-05-26 12:34:56.789123";
    /// Start of a partial slice, and the default tile boundary.
    const SLICE_START: i64 = 1_791_547_200_000;

    fn temp_store() -> (TempDir, CohortStore) {
        let dir = TempDir::new().unwrap();
        let store = CohortStore::open(&StoreConfig {
            path: dir.path().join("db"),
            ..StoreConfig::default()
        })
        .unwrap();
        (dir, store)
    }

    fn store_handle(store: &CohortStore) -> StoreHandle {
        StoreHandle::new(
            store.clone(),
            OffloadConfig {
                mode: OffloadMode::All,
                event_read_permits: 16,
                maintenance_permits: 6,
            },
        )
    }

    fn behavioral_leaf() -> Value {
        json!({
            "type": "behavioral", "value": "performed_event", "key": "$pageview",
            "time_value": 7, "time_interval": "day",
            "conditionHash": "0123456789abcdef",
            "bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
        })
    }

    fn person_leaf() -> Value {
        json!({
            "type": "person", "key": "email", "value": "[email protected]", "operator": "exact",
            "conditionHash": "fedcba9876543210",
            "bytecode": ["_H", 1, 32, "[email protected]", 32, "email", 32, "properties", 32, "person", 1, 3, 11],
        })
    }

    /// `behavioral_hash` is the catalog's behavioral shape hash for the cohort, `None` for one it
    /// has not read yet.
    fn cohort_filters(single_leaf: bool, behavioral_hash: Option<&str>) -> TeamFilters {
        let values = if single_leaf {
            vec![behavioral_leaf()]
        } else {
            vec![behavioral_leaf(), person_leaf()]
        };
        let cohort = json!({ "properties": { "type": "AND", "values": values } });
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(CohortId(COHORT), TeamId(TEAM), &cohort)
            .unwrap();
        if let Some(hash) = behavioral_hash {
            builder.set_behavioral_shape_hash(
                CohortId(COHORT),
                BehavioralShapeHash::parse(hash).unwrap(),
            );
        }
        if !single_leaf {
            // A mixed cohort carries both guards, so either kind of run can reconcile it.
            builder.set_person_shape_hash(
                CohortId(COHORT),
                PersonShapeHash::parse(SHAPE_HASH).unwrap(),
            );
        }
        builder.freeze(UTC)
    }

    fn catalog(single_leaf: bool) -> (Arc<CatalogHandle>, LeafStateKey) {
        catalog_over(cohort_filters(single_leaf, Some(SHAPE_HASH)))
    }

    fn catalog_over(filters: TeamFilters) -> (Arc<CatalogHandle>, LeafStateKey) {
        let lsk = filters.by_condition_to_lsk[&BEHAVIORAL_HASH][0];
        (handle_for(filters), lsk)
    }

    /// A person-only cohort: no behavioral leaf, so only the person guard fences its reconcile.
    fn person_catalog() -> (Arc<CatalogHandle>, LeafStateKey) {
        let cohort = json!({ "properties": { "type": "AND", "values": [person_leaf()] } });
        let mut builder = TeamFiltersBuilder::default();
        builder
            .add_cohort(CohortId(COHORT), TeamId(TEAM), &cohort)
            .unwrap();
        builder.set_person_shape_hash(
            CohortId(COHORT),
            PersonShapeHash::parse(SHAPE_HASH).unwrap(),
        );
        (
            handle_for(builder.freeze(UTC)),
            LeafStateKey::for_person_property(&PERSON_HASH),
        )
    }

    fn unloaded() -> Arc<CatalogHandle> {
        Arc::new(CatalogHandle::new())
    }

    fn team_catalog(filters: TeamFilters) -> FilterCatalog {
        FilterCatalog::from_teams([(TeamId(TEAM), filters)])
    }

    fn handle_for(filters: TeamFilters) -> Arc<CatalogHandle> {
        Arc::new(CatalogHandle::from_catalog(team_catalog(filters)))
    }

    struct DrainShell {
        _dir: TempDir,
        store: CohortStore,
        handle: StoreHandle,
        catalog: Arc<CatalogHandle>,
        sink: Arc<dyn MembershipSink>,
        markers: CaptureReconcileMarkerSink,
        deps: MergeWorkerDeps,
        queue: ReconcileQueue,
        lsk: LeafStateKey,
    }

    impl DrainShell {
        fn new(
            single_leaf: bool,
            sink: Arc<dyn MembershipSink>,
            cascade_sink: CaptureCascadeSink,
            scan_page: usize,
        ) -> Self {
            Self::over(catalog(single_leaf), sink, cascade_sink, scan_page)
        }

        fn person_only(
            sink: Arc<dyn MembershipSink>,
            cascade_sink: CaptureCascadeSink,
            scan_page: usize,
        ) -> Self {
            Self::over(person_catalog(), sink, cascade_sink, scan_page)
        }

        fn over(
            (catalog, lsk): (Arc<CatalogHandle>, LeafStateKey),
            sink: Arc<dyn MembershipSink>,
            cascade_sink: CaptureCascadeSink,
            scan_page: usize,
        ) -> Self {
            let (_dir, store) = temp_store();
            let handle = store_handle(&store);
            let mut deps = Arc::try_unwrap(MergeWorkerDeps::capture())
                .unwrap_or_else(|_| panic!("test owns the only dependency Arc"));
            deps.reconcile.enabled = true;
            deps.reconcile.scan_page = scan_page;
            deps.cascade.enabled = true;
            deps.cascade_sink = Arc::new(cascade_sink);
            let markers = CaptureReconcileMarkerSink::new();
            deps.reconcile.marker_sink = Arc::new(markers.clone());
            let queue = ReconcileQueue::new(
                PARTITION,
                deps.reconcile.backlog.clone(),
                handle.clone(),
                catalog.clone(),
                SliceCoverage::Complete,
            );
            Self {
                _dir,
                store,
                handle,
                catalog,
                sink,
                markers,
                deps,
                queue,
                lsk,
            }
        }

        /// Replaces the queue, so call it before enqueueing.
        fn cover(&mut self, coverage: SliceCoverage) {
            assert_eq!(self.queue.len(), 0, "covering replaces the queue");
            self.queue = ReconcileQueue::new(
                PARTITION,
                self.deps.reconcile.backlog.clone(),
                self.handle.clone(),
                self.catalog.clone(),
                coverage,
            );
        }

        /// Re-arm the marker sink to fail its first produce, isolating the marker phase's retry from
        /// the membership leg's.
        fn fail_first_marker(&mut self) {
            self.markers = CaptureReconcileMarkerSink::failing_first(1);
            self.deps.reconcile.marker_sink = Arc::new(self.markers.clone());
        }

        fn write_current(&self, person: Uuid, behavioral: bool, person_match: bool, prior: bool) {
            if behavioral {
                let key = BehavioralKey::new(PARTITION, TEAM as u64, person, self.lsk);
                let state = Stage1State::BehavioralSingle {
                    has_match: true,
                    last_event_at_ms: 1_700_000_000_000,
                    earliest_eviction_at_ms: i64::MAX,
                };
                let record = StatefulRecord::new(state, AppliedOffsets::default());
                self.store
                    .write_batch(|batch| batch.put::<Behavioral>(&key, &record.encode()))
                    .unwrap();
            }
            if person_match {
                let key = PersonRecordKey::new(PARTITION, TEAM as u64, person);
                let mut record = PersonRecord::absent();
                record.matched = MatchedSet::from_iter([PERSON_HASH]);
                self.store
                    .write_batch(|batch| batch.put::<PersonRecords>(&key, &record.encode()))
                    .unwrap();
            }
            let key = Stage2Key {
                partition_id: PARTITION,
                team_id: TEAM as u64,
                cohort_id: COHORT as u64,
                person_id: person,
            };
            let state = Stage2State {
                in_cohort: prior,
                last_evaluated_at_ms: 1,
            };
            self.store
                .write_batch(|batch| batch.put_stage2(&key, &state.encode()))
                .unwrap();
        }

        fn stage2_bit(&self, person: Uuid) -> bool {
            let key = Stage2Key {
                partition_id: PARTITION,
                team_id: TEAM as u64,
                cohort_id: COHORT as u64,
                person_id: person,
            };
            Stage2State::decode(&self.store.get_stage2(&key).unwrap().unwrap())
                .unwrap()
                .in_cohort
        }

        fn enqueue(&mut self, tile: ReconcileTile, offset: i64) {
            self.deps
                .seed_tracker
                .mark_dispatched(PARTITION as i32, offset + 1);
            if self.committable().is_none() {
                assert_eq!(
                    self.deps
                        .seed_tracker
                        .mark_processed(PARTITION as i32, offset),
                    MarkOutcome::WithinDispatch,
                );
            }
            let deferred = self.deps.seed_tracker.defer(PARTITION as i32, offset);
            self.queue.enqueue(tile, deferred);
        }

        /// Admit a newer run over a queued job exactly as `admit_reconcile` does: supersede, hand the
        /// superseded floor to `replace_deferred`, then enqueue the replacement at the new offset.
        fn supersede(&mut self, tile: ReconcileTile, offset: i64) {
            self.deps
                .seed_tracker
                .mark_dispatched(PARTITION as i32, offset + 1);
            let SupersedeOutcome::Replaced(superseded) = self.queue.supersede_if_newer(
                tile.team_id(),
                tile.cohort_id(),
                tile.scope().kind(),
                offset,
            ) else {
                panic!("a newer run must supersede the queued job");
            };
            let (replacement, _) = self
                .deps
                .seed_tracker
                .replace_deferred(superseded, offset)
                .expect("the superseded reconcile retains its deferred offset");
            self.queue.enqueue(tile, replacement);
        }

        async fn tick(&mut self) {
            handle_reconcile_drain(
                PARTITION,
                &self.handle,
                &self.sink,
                &self.deps,
                &mut self.queue,
                TS,
            )
            .await;
        }

        fn committable(&self) -> Option<i64> {
            self.deps
                .seed_tracker
                .committable_offsets()
                .get(&(PARTITION as i32))
                .copied()
        }
    }

    fn tile(team_id: i32, cohort_id: i32, run_id: u128) -> ReconcileTile {
        scoped_tile(team_id, cohort_id, run_id, ScopeKind::Behavioral)
    }

    fn scoped_tile(team_id: i32, cohort_id: i32, run_id: u128, guard: ScopeKind) -> ReconcileTile {
        bounded_tile(team_id, cohort_id, run_id, guard, SLICE_START)
    }

    fn bounded_tile(
        team_id: i32,
        cohort_id: i32,
        run_id: u128,
        guard: ScopeKind,
        boundary: i64,
    ) -> ReconcileTile {
        ReconcileTile::new(
            TeamId(team_id),
            CohortId(cohort_id),
            ReconcileScope::parse(guard, SHAPE_HASH).unwrap(),
            RunId(Uuid::from_u128(run_id)),
            RunBoundaryMs(boundary),
        )
    }

    #[test]
    fn queue_tracks_backlog_and_drop_releases_only_the_backlog_count() {
        let (_dir, store) = temp_store();
        let tracker = OffsetTracker::new();
        let backlog = Arc::new(ReconcileBacklog::default());
        tracker.mark_dispatched(3, 11);

        {
            let mut queue = ReconcileQueue::new(
                3,
                backlog.clone(),
                store_handle(&store),
                unloaded(),
                SliceCoverage::Complete,
            );
            queue.enqueue(tile(1, 7, 1), tracker.defer(3, 10));
            assert_eq!(queue.len(), 1);
            assert_eq!(backlog.len(), 1);
        }

        assert!(backlog.is_empty());
        assert_eq!(
            tracker.mark_processed(3, 11),
            crate::partitions::offset_tracker::MarkOutcome::WithinDispatch,
        );
        assert_eq!(
            tracker.committable_offsets().get(&3),
            Some(&10),
            "dropping a queued job does not complete its deferred offset",
        );
    }

    #[test]
    fn only_the_active_queue_head_tracks_dirty_people() {
        let (_dir, store) = temp_store();
        let tracker = OffsetTracker::new();
        let backlog = Arc::new(ReconcileBacklog::default());
        tracker.mark_dispatched(PARTITION as i32, 12);
        let mut queue = ReconcileQueue::new(
            PARTITION,
            backlog,
            store_handle(&store),
            unloaded(),
            SliceCoverage::Complete,
        );
        queue.enqueue(tile(TEAM, COHORT, 1), tracker.defer(PARTITION as i32, 10));
        queue.enqueue(
            tile(TEAM, COHORT + 1, 2),
            tracker.defer(PARTITION as i32, 11),
        );
        let head_prefix = Stage2CohortPrefix {
            partition_id: PARTITION,
            team_id: TEAM as u64,
            cohort_id: COHORT as u64,
        };
        let waiting_prefix = Stage2CohortPrefix {
            cohort_id: (COHORT + 1) as u64,
            ..head_prefix
        };

        // Enqueue alone captures nothing: a future full scan covers these writes.
        for (prefix, person) in [(head_prefix, 1), (waiting_prefix, 2)] {
            let key = Stage2Key {
                partition_id: prefix.partition_id,
                team_id: prefix.team_id,
                cohort_id: prefix.cohort_id,
                person_id: Uuid::from_u128(person),
            };
            store
                .write_batch(|batch| batch.put_stage2(&key, b"before"))
                .unwrap();
            assert!(store
                .scan_stage2_dirty(prefix, None, 10)
                .unwrap()
                .is_empty());
        }

        queue.activate_front_tracking(head_prefix);
        for (prefix, person) in [(head_prefix, 3), (waiting_prefix, 4)] {
            let key = Stage2Key {
                partition_id: prefix.partition_id,
                team_id: prefix.team_id,
                cohort_id: prefix.cohort_id,
                person_id: Uuid::from_u128(person),
            };
            store
                .write_batch(|batch| batch.put_stage2(&key, b"during"))
                .unwrap();
        }

        assert_eq!(
            store
                .scan_stage2_dirty(head_prefix, None, 10)
                .unwrap()
                .len(),
            1
        );
        assert!(store
            .scan_stage2_dirty(waiting_prefix, None, 10)
            .unwrap()
            .is_empty());
    }

    #[test]
    fn supersede_matches_team_and_cohort_and_preserves_other_jobs() {
        let (_dir, store) = temp_store();
        let tracker = OffsetTracker::new();
        let backlog = Arc::new(ReconcileBacklog::default());
        tracker.mark_dispatched(3, 14);
        let mut queue = ReconcileQueue::new(
            3,
            backlog.clone(),
            store_handle(&store),
            unloaded(),
            SliceCoverage::Complete,
        );
        queue.enqueue(tile(1, 7, 1), tracker.defer(3, 10));
        queue.enqueue(tile(1, 8, 2), tracker.defer(3, 11));
        queue.enqueue(tile(2, 7, 3), tracker.defer(3, 12));

        let SupersedeOutcome::Replaced(superseded) =
            queue.supersede_if_newer(TeamId(1), CohortId(7), ScopeKind::Behavioral, 13)
        else {
            panic!("the newer run should supersede team 1 cohort 7");
        };

        assert_eq!(queue.len(), 2);
        assert_eq!(queue.front_run_id(), Some(RunId(Uuid::from_u128(2))));
        assert_eq!(backlog.len(), 2);
        assert_eq!(
            tracker.complete_deferred(superseded),
            Some(crate::partitions::offset_tracker::MarkOutcome::WithinDispatch),
        );
        assert!(matches!(
            queue.supersede_if_newer(TeamId(2), CohortId(7), ScopeKind::Behavioral, 12),
            SupersedeOutcome::RetainedNewerOrEqual,
        ));
    }

    #[test]
    fn a_mixed_cohorts_two_guards_queue_side_by_side_and_supersede_only_their_own() {
        let (_dir, store) = temp_store();
        let tracker = OffsetTracker::new();
        let backlog = Arc::new(ReconcileBacklog::default());
        tracker.mark_dispatched(3, 20);
        let mut queue = ReconcileQueue::new(
            3,
            backlog.clone(),
            store_handle(&store),
            unloaded(),
            SliceCoverage::Complete,
        );
        queue.enqueue(
            scoped_tile(TEAM, COHORT, 1, ScopeKind::Behavioral),
            tracker.defer(3, 10),
        );
        queue.enqueue(
            scoped_tile(TEAM, COHORT, 2, ScopeKind::PersonProperty),
            tracker.defer(3, 11),
        );

        // A newer person run evicts only the queued person job; the behavioral snapshot is a
        // different fence and still owes its own marker.
        let SupersedeOutcome::Replaced(superseded) = queue.supersede_if_newer(
            TeamId(TEAM),
            CohortId(COHORT),
            ScopeKind::PersonProperty,
            12,
        ) else {
            panic!("the newer person run should supersede the queued person job");
        };
        assert_eq!(queue.len(), 1);
        assert_eq!(queue.front_run_id(), Some(RunId(Uuid::from_u128(1))));
        tracker.complete_deferred(superseded);
        assert_eq!(
            tracker.committable_offsets().get(&3),
            Some(&10),
            "the surviving behavioral job still pins the partition's floor at its own offset",
        );

        assert!(
            matches!(
                queue.supersede_if_newer(
                    TeamId(TEAM),
                    CohortId(COHORT),
                    ScopeKind::PersonProperty,
                    13,
                ),
                SupersedeOutcome::NoQueuedJob,
            ),
            "the surviving behavioral job is invisible to a person-kind supersession",
        );
    }

    #[test]
    fn older_or_duplicate_replay_never_evicts_the_queued_run() {
        let (_dir, store) = temp_store();
        let tracker = OffsetTracker::new();
        let backlog = Arc::new(ReconcileBacklog::default());
        tracker.mark_dispatched(3, 20);
        let mut queue = ReconcileQueue::new(
            3,
            backlog.clone(),
            store_handle(&store),
            unloaded(),
            SliceCoverage::Complete,
        );
        queue.enqueue(tile(1, 7, 2), tracker.defer(3, 15));

        for replayed_offset in [10, 15] {
            assert!(matches!(
                queue.supersede_if_newer(
                    TeamId(1),
                    CohortId(7),
                    ScopeKind::Behavioral,
                    replayed_offset,
                ),
                SupersedeOutcome::RetainedNewerOrEqual,
            ));
            assert_eq!(queue.front_run_id(), Some(RunId(Uuid::from_u128(2))));
            assert_eq!(backlog.len(), 1);
            assert_eq!(tracker.committable_offsets().get(&3), None);
        }
    }

    #[tokio::test]
    async fn superseding_a_mid_drain_run_reemits_under_the_new_run_and_releases_its_offset() {
        let sink = CaptureSink::new();
        // A one-row page over three people keeps run A mid-scan (active dirty lease, cursor set) when
        // the newer run arrives.
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
        let people = [Uuid::from_u128(1), Uuid::from_u128(2), Uuid::from_u128(3)];
        for person in people {
            shell.write_current(person, true, false, true);
        }
        shell.enqueue(tile(TEAM, COHORT, 100), 5);

        shell.tick().await;
        assert_eq!(
            sink.changes().len(),
            1,
            "run A emitted one page before being superseded",
        );
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::Scanning { cursor: Some(_) }),
        ));

        shell.supersede(tile(TEAM, COHORT, 200), 7);
        assert_eq!(shell.queue.len(), 1);
        assert_eq!(
            shell.queue.front_run_id(),
            Some(RunId(Uuid::from_u128(200)))
        );

        for _ in 0..16 {
            if shell.queue.front().is_none() {
                break;
            }
            shell.tick().await;
        }
        assert!(shell.queue.front().is_none());
        assert!(shell.deps.reconcile.backlog.is_empty());

        let markers = shell.markers.markers();
        assert_eq!(
            markers.len(),
            1,
            "the superseded run never emits its own marker"
        );
        assert_eq!(markers[0].run_id(), RunId(Uuid::from_u128(200)));

        let reemitted: std::collections::HashSet<String> = sink
            .changes()
            .iter()
            .filter(|change| change.run_id == Some(RunId(Uuid::from_u128(200))))
            .map(|change| change.person_id.clone())
            .collect();
        assert_eq!(
            reemitted,
            people
                .iter()
                .map(Uuid::to_string)
                .collect::<std::collections::HashSet<_>>(),
            "run B re-emits the full current membership",
        );

        assert_eq!(
            shell.committable(),
            Some(8),
            "the deferred floor releases only after run B completes",
        );
    }

    fn guard_filters(
        eligibility: Option<CohortEligibility>,
        guards: &[(ScopeKind, &str)],
    ) -> TeamFilters {
        let mut filters = TeamFilters::default();
        filters.cohorts.insert(
            CohortId(COHORT),
            CohortTree {
                cohort_id: CohortId(COHORT),
                team_id: TeamId(TEAM),
                root: FilterNode::Group {
                    op: BoolOp::And,
                    children: Vec::new(),
                },
            },
        );
        if let Some(eligibility) = eligibility {
            filters.eligibility.insert(CohortId(COHORT), eligibility);
        }
        for &(kind, hash) in guards {
            match kind {
                ScopeKind::Behavioral => {
                    filters
                        .behavioral_shape_hashes
                        .insert(CohortId(COHORT), BehavioralShapeHash::parse(hash).unwrap());
                }
                ScopeKind::PersonProperty => {
                    filters
                        .person_shape_hashes
                        .insert(CohortId(COHORT), PersonShapeHash::parse(hash).unwrap());
                }
            }
        }
        filters
    }

    fn verdict(
        filters: TeamFilters,
        tile: &ReconcileTile,
    ) -> Result<CohortEligibility, CatalogMiss> {
        catalog_verdict(Some(&Arc::new(filters)), tile).map(|(_, eligibility)| eligibility)
    }

    #[test]
    fn catalog_verdict_names_every_discard_reason_and_proves_what_proceeds() {
        let reconcile = tile(TEAM, COHORT, 1);

        assert_eq!(
            catalog_verdict(None, &reconcile).map(|(_, eligibility)| eligibility),
            Err(CatalogMiss::TeamAbsent),
        );
        assert_eq!(
            verdict(TeamFilters::default(), &reconcile),
            Err(CatalogMiss::CohortAbsent),
        );

        let behavioral_guard = &[(ScopeKind::Behavioral, SHAPE_HASH)];
        assert_eq!(
            verdict(guard_filters(None, behavioral_guard), &reconcile),
            Err(CatalogMiss::CohortAbsent),
        );
        let excluded = guard_filters(
            Some(CohortEligibility::Excluded(ExcludedReason::HasDroppedLeaf)),
            behavioral_guard,
        );
        assert_eq!(verdict(excluded, &reconcile), Err(CatalogMiss::NotEmitting),);

        for eligibility in [
            CohortEligibility::SingleLeaf(LeafStateKey(BEHAVIORAL_HASH)),
            CohortEligibility::Stage2Composable,
            CohortEligibility::Stage2ComposableRef,
        ] {
            assert_eq!(
                verdict(
                    guard_filters(Some(eligibility), behavioral_guard),
                    &reconcile
                ),
                Ok(eligibility),
                "{eligibility:?} registers membership",
            );
        }
    }

    #[test]
    fn each_guard_kind_consults_only_its_own_map() {
        let emitting = Some(CohortEligibility::Stage2Composable);
        for (guard, other) in [
            (ScopeKind::Behavioral, ScopeKind::PersonProperty),
            (ScopeKind::PersonProperty, ScopeKind::Behavioral),
        ] {
            let reconcile = scoped_tile(TEAM, COHORT, 1, guard);
            assert_eq!(
                verdict(guard_filters(emitting, &[(guard, SHAPE_HASH)]), &reconcile),
                Ok(CohortEligibility::Stage2Composable),
                "{guard} matches its own map",
            );
            assert_eq!(
                verdict(guard_filters(emitting, &[(guard, "shape-v2")]), &reconcile),
                Err(CatalogMiss::HashMismatch),
                "{guard} diverged",
            );
            assert_eq!(
                verdict(guard_filters(emitting, &[]), &reconcile),
                Err(CatalogMiss::HashUnknown),
                "{guard} has no persisted hash",
            );
            // The catalog carries only the *other* kind's hash, at the very value the tile pins.
            assert_eq!(
                verdict(guard_filters(emitting, &[(other, SHAPE_HASH)]), &reconcile),
                Err(CatalogMiss::HashUnknown),
                "{guard} must not be satisfied by the {other} map",
            );
        }
    }

    /// The guard's decision without the filters it carries, so cases compare by value.
    #[derive(Debug, PartialEq, Eq)]
    enum Fate {
        Proceed(CohortEligibility),
        Retry(ReconcileRetryReason),
        Withhold(WithheldReason),
    }

    impl From<ReconcileGuard> for Fate {
        fn from(guard: ReconcileGuard) -> Self {
            match guard {
                ReconcileGuard::Proceed { eligibility, .. } => Self::Proceed(eligibility),
                ReconcileGuard::Retry(reason) => Self::Retry(reason),
                ReconcileGuard::Withhold(reason) => Self::Withhold(reason),
            }
        }
    }

    #[derive(Debug, Clone, Copy)]
    enum Freshness {
        NotLoaded,
        RefreshedBeforeAdmission,
        RefreshedAfterAdmission,
    }

    /// What the refresh the case publishes holds for the job's cohort.
    #[derive(Debug, Clone, Copy)]
    enum Snapshot {
        Emitting,
        MissingTeam,
    }

    impl Snapshot {
        fn catalog(self) -> FilterCatalog {
            match self {
                Self::Emitting => team_catalog(guard_filters(
                    Some(CohortEligibility::Stage2Composable),
                    &[(ScopeKind::Behavioral, SHAPE_HASH)],
                )),
                Self::MissingTeam => FilterCatalog::from_teams([]),
            }
        }
    }

    #[derive(Debug, Clone, Copy)]
    enum Slice {
        Complete,
        BeganAtBoundary,
        BeganAfterBoundary,
    }

    impl Slice {
        fn coverage(self) -> SliceCoverage {
            match self {
                Self::Complete => SliceCoverage::Complete,
                Self::BeganAtBoundary | Self::BeganAfterBoundary => {
                    SliceCoverage::Since(CoverageStartMs(SLICE_START))
                }
            }
        }

        fn run_boundary(self) -> i64 {
            match self {
                Self::BeganAtBoundary => SLICE_START,
                Self::Complete | Self::BeganAfterBoundary => SLICE_START - 1,
            }
        }
    }

    #[test]
    fn a_catalog_miss_withholds_only_after_a_later_refresh_and_coverage_only_after_the_catalog() {
        let team_absent = CatalogMiss::TeamAbsent;
        let partial = WithheldReason::PartialCoverage {
            covered_since: CoverageStartMs(SLICE_START),
        };
        let cases = [
            // Nothing is published before the first load, so the snapshot kind does not matter.
            (
                Freshness::NotLoaded,
                Snapshot::MissingTeam,
                Slice::BeganAfterBoundary,
                Fate::Retry(ReconcileRetryReason::CatalogNotLoaded),
            ),
            (
                Freshness::RefreshedBeforeAdmission,
                Snapshot::Emitting,
                Slice::Complete,
                Fate::Proceed(CohortEligibility::Stage2Composable),
            ),
            // The catalog wait comes before coverage.
            (
                Freshness::RefreshedBeforeAdmission,
                Snapshot::MissingTeam,
                Slice::BeganAfterBoundary,
                Fate::Retry(ReconcileRetryReason::AwaitingRefresh(team_absent)),
            ),
            (
                Freshness::RefreshedAfterAdmission,
                Snapshot::Emitting,
                Slice::Complete,
                Fate::Proceed(CohortEligibility::Stage2Composable),
            ),
            (
                Freshness::RefreshedAfterAdmission,
                Snapshot::Emitting,
                Slice::BeganAtBoundary,
                Fate::Proceed(CohortEligibility::Stage2Composable),
            ),
            (
                Freshness::RefreshedAfterAdmission,
                Snapshot::Emitting,
                Slice::BeganAfterBoundary,
                Fate::Withhold(partial),
            ),
            // The catalog's reason wins over coverage.
            (
                Freshness::RefreshedAfterAdmission,
                Snapshot::MissingTeam,
                Slice::BeganAfterBoundary,
                Fate::Withhold(WithheldReason::Catalog(team_absent)),
            ),
        ];

        for (freshness, snapshot, slice, expected) in cases {
            let reconcile =
                bounded_tile(TEAM, COHORT, 1, ScopeKind::Behavioral, slice.run_boundary());
            let catalog = CatalogHandle::new();
            let admitted_after = match freshness {
                Freshness::NotLoaded => catalog.last_begun(),
                Freshness::RefreshedBeforeAdmission => {
                    catalog.publish(snapshot.catalog());
                    catalog.last_begun()
                }
                Freshness::RefreshedAfterAdmission => {
                    let admitted_after = catalog.last_begun();
                    catalog.publish(snapshot.catalog());
                    admitted_after
                }
            };

            assert_eq!(
                Fate::from(evaluate_guard(
                    &catalog.view(),
                    admitted_after,
                    &reconcile,
                    slice.coverage(),
                )),
                expected,
                "{freshness:?} with {snapshot:?} on a {slice:?} slice",
            );
        }
    }

    fn counted(metrics: &metrics_exporter_prometheus::PrometheusHandle, series: &str) -> u64 {
        metrics
            .render()
            .lines()
            .filter(|line| !line.starts_with('#'))
            .find(|line| series.split(',').all(|part| line.contains(part)))
            .and_then(|line| line.rsplit(' ').next())
            .map_or(0, |count| count.parse().unwrap())
    }

    /// A run's reconcile can reach the processor before the catalog refresh that reads the run's
    /// pinned hash. The job waits with its seed offset held, and the next refresh decides it.
    #[tokio::test]
    async fn a_reconcile_that_beats_its_catalog_refresh_waits_for_the_next_one() {
        for (refreshed_hash, completes) in [(SHAPE_HASH, true), ("shape-v2", false)] {
            let recorder = metrics_exporter_prometheus::PrometheusBuilder::new().build_recorder();
            let metrics = recorder.handle();
            let _local = metrics::set_default_local_recorder(&recorder);
            let mut shell = DrainShell::over(
                catalog_over(cohort_filters(true, None)),
                Arc::new(CaptureSink::new()),
                CaptureCascadeSink::new(),
                8,
            );
            // A refresh in flight at admission may have read Postgres before the run's write, so
            // its snapshot cannot discard the job either.
            let catalog = shell.catalog.clone();
            let in_flight = catalog.begin_refresh();
            shell.enqueue(tile(TEAM, COHORT, 1), 5);
            in_flight.publish(team_catalog(cohort_filters(true, None)));

            shell.tick().await;
            shell.tick().await;
            assert!(shell.markers.markers().is_empty(), "{refreshed_hash}");
            assert_eq!(shell.queue.len(), 1, "the job waits for the next refresh");
            assert_eq!(
                shell.committable(),
                Some(5),
                "its seed offset stays deferred"
            );
            assert_eq!(
                counted(
                    &metrics,
                    "cohort_reconcile_jobs_deferred_total,reason=\"hash_unknown\""
                ),
                1,
                "a wait over several ticks counts once",
            );

            shell
                .catalog
                .publish(team_catalog(cohort_filters(true, Some(refreshed_hash))));
            shell.tick().await;

            let markers = shell.markers.markers();
            assert_eq!(markers.len(), 1, "{refreshed_hash}");
            assert_eq!(markers[0].run_id(), RunId(Uuid::from_u128(1)));
            if completes {
                assert!(matches!(markers[0], ReconcileMarker::Complete(_)));
            } else {
                // A superseded run fails closed with a withheld marker.
                assert!(
                    matches!(
                        &markers[0],
                        ReconcileMarker::Withheld(withheld)
                            if withheld.reason()
                                == WithheldReason::Catalog(CatalogMiss::HashMismatch)
                    ),
                    "{markers:?}",
                );
                assert_eq!(
                    counted(
                        &metrics,
                        "cohort_reconcile_jobs_discarded_total,reason=\"hash_mismatch\""
                    ),
                    1,
                );
            }
            assert!(shell.queue.front().is_none(), "{refreshed_hash}");
            assert_eq!(shell.committable(), Some(6), "{refreshed_hash}");
        }
    }

    #[tokio::test]
    async fn drain_emits_the_full_snapshot_fixes_only_flips_and_paginates() {
        let sink = CaptureSink::new();
        let cascades = CaptureCascadeSink::new();
        let mut shell = DrainShell::new(false, Arc::new(sink.clone()), cascades.clone(), 2);
        let alice = Uuid::from_u128(1);
        let bob = Uuid::from_u128(2);
        let charlie = Uuid::from_u128(3);
        shell.write_current(alice, true, true, false);
        shell.write_current(bob, false, false, true);
        shell.write_current(charlie, true, true, true);
        shell.enqueue(tile(TEAM, COHORT, 11), 5);

        shell.tick().await;

        assert_eq!(sink.changes().len(), 2, "one bounded page was emitted");
        assert!(shell.markers.markers().is_empty());
        assert_eq!(shell.committable(), Some(5));
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::Scanning {
                cursor: Some(_),
                ..
            }),
        ));

        shell.tick().await;

        let changes = sink.changes();
        assert_eq!(changes.len(), 3, "unchanged rows are emitted too");
        assert_eq!(
            changes
                .iter()
                .map(|change| change.status)
                .collect::<Vec<_>>(),
            vec![
                MembershipStatus::Entered,
                MembershipStatus::Left,
                MembershipStatus::Entered,
            ],
        );
        assert!(changes.iter().all(|change| {
            change.origin == Some(ChangeOrigin::Reconcile)
                && change.run_id == Some(RunId(Uuid::from_u128(11)))
        }));
        assert!(shell.stage2_bit(alice));
        assert!(!shell.stage2_bit(bob));
        assert!(shell.stage2_bit(charlie));

        let cascade_messages = cascades.messages();
        assert_eq!(cascade_messages.len(), 2, "only flipped bits cascade");
        assert_eq!(cascade_messages[0].change.person_id, alice.to_string());
        assert_eq!(cascade_messages[1].change.person_id, bob.to_string());
        shell.tick().await;
        assert_eq!(shell.markers.markers().len(), 1);
        assert!(shell.queue.front().is_none());
        assert!(shell.deps.reconcile.backlog.is_empty());
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn one_cohorts_two_guarded_jobs_each_drain_to_their_own_marker() {
        let sink = CaptureSink::new();
        let mut shell =
            DrainShell::new(false, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
        shell.write_current(Uuid::from_u128(1), true, true, false);
        shell.enqueue(scoped_tile(TEAM, COHORT, 31, ScopeKind::Behavioral), 5);
        shell.enqueue(scoped_tile(TEAM, COHORT, 32, ScopeKind::PersonProperty), 6);

        for _ in 0..8 {
            if shell.queue.front().is_none() {
                break;
            }
            shell.tick().await;
        }

        // Both jobs lease the same Stage 2 prefix in turn; neither may release the other's capture.
        assert_eq!(
            shell
                .markers
                .markers()
                .iter()
                .map(ReconcileMarker::run_id)
                .collect::<Vec<_>>(),
            vec![RunId(Uuid::from_u128(31)), RunId(Uuid::from_u128(32))],
        );
        assert!(shell.deps.reconcile.backlog.is_empty());
        assert_eq!(shell.committable(), Some(7));
    }

    #[tokio::test]
    async fn a_person_only_cohort_drains_under_its_person_guard_and_marks_complete() {
        let sink = CaptureSink::new();
        let mut shell =
            DrainShell::person_only(Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
        let matching = Uuid::from_u128(1);
        let left = Uuid::from_u128(2);
        shell.write_current(matching, false, true, false);
        shell.write_current(left, false, false, true);
        shell.enqueue(scoped_tile(TEAM, COHORT, 21, ScopeKind::PersonProperty), 5);

        shell.tick().await;
        shell.tick().await;

        let statuses: Vec<_> = sink
            .changes()
            .iter()
            .map(|change| (change.person_id.clone(), change.status))
            .collect();
        assert_eq!(
            statuses,
            vec![
                (matching.to_string(), MembershipStatus::Entered),
                (left.to_string(), MembershipStatus::Left),
            ],
            "the snapshot is computed from person state, not a behavioral leaf",
        );
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(
            shell.markers.markers()[0].run_id(),
            RunId(Uuid::from_u128(21))
        );
        assert!(shell.queue.front().is_none());
    }

    #[tokio::test]
    async fn noop_reconcile_claims_a_transferred_fallback() {
        let sink = CaptureSink::new();
        let cascades = CaptureCascadeSink::new();
        let mut shell = DrainShell::new(false, Arc::new(sink.clone()), cascades.clone(), 2);
        let person = Uuid::from_u128(1);
        let key = Stage2Key {
            partition_id: PARTITION,
            team_id: TEAM as u64,
            cohort_id: COHORT as u64,
            person_id: person,
        };
        let fallback = Stage2State {
            in_cohort: false,
            last_evaluated_at_ms: 1,
        };
        shell
            .store
            .write_batch(|batch| {
                batch.put_stage2(&key, &fallback.encode_transferred_fallback());
            })
            .unwrap();
        shell.enqueue(tile(TEAM, COHORT, 12), 5);

        shell.tick().await;

        let bytes = shell.store.get_stage2(&key).unwrap().unwrap();
        let (state, ownership) = Stage2State::decode_with_ownership(&bytes).unwrap();
        assert!(!state.in_cohort);
        assert_eq!(ownership, Stage2Ownership::Local);
        assert_eq!(
            sink.changes().len(),
            1,
            "the full snapshot still emits the row"
        );
        assert_eq!(sink.changes()[0].status, MembershipStatus::Left);
        assert!(
            cascades.messages().is_empty(),
            "ownership settlement is not a flip"
        );
    }

    #[tokio::test]
    async fn paged_scan_drains_a_row_inserted_behind_its_cursor_without_restarting() {
        let sink = CaptureSink::new();
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
        let later_key = Uuid::from_u128(2);
        let inserted_behind_cursor = Uuid::from_u128(1);
        shell.write_current(later_key, true, false, true);
        shell.enqueue(tile(TEAM, COHORT, 111), 5);

        shell.tick().await;
        assert_eq!(sink.changes().len(), 1);
        assert_eq!(sink.changes()[0].person_id, later_key.to_string());
        assert!(shell.markers.markers().is_empty());

        shell.write_current(inserted_behind_cursor, true, false, true);
        shell.tick().await;

        assert_eq!(
            sink.changes()[1].person_id,
            inserted_behind_cursor.to_string(),
            "the dirty-person pass catches a row inserted behind the main cursor",
        );
        assert_eq!(
            sink.changes()
                .iter()
                .map(|change| change.person_id.clone())
                .collect::<Vec<_>>(),
            vec![later_key.to_string(), inserted_behind_cursor.to_string(),],
            "the main cohort page is not rescanned because another person changed",
        );
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn repeatedly_mutated_hot_person_does_not_restart_main_scan() {
        let sink = CaptureSink::new();
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
        let hot = Uuid::from_u128(10);
        let second = Uuid::from_u128(20);
        let third = Uuid::from_u128(30);
        for person in [hot, second, third] {
            shell.write_current(person, true, false, true);
        }
        shell.enqueue(tile(TEAM, COHORT, 112), 5);

        for expected in [hot, second, third] {
            shell.tick().await;
            assert_eq!(
                sink.changes().last().unwrap().person_id,
                expected.to_string(),
                "the main cursor advances despite a mutation behind it",
            );
            shell.write_current(hot, true, false, true);
        }

        shell.tick().await;
        assert_eq!(sink.changes().last().unwrap().person_id, hot.to_string());
        assert_eq!(
            shell.markers.markers().len(),
            1,
            "the job completes in the same tick that settles the full hot-person page",
        );
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn deleted_dirty_row_is_cleared_without_being_resurrected_or_emitted_again() {
        let sink = CaptureSink::new();
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
        let alice = Uuid::from_u128(1);
        let bob = Uuid::from_u128(2);
        shell.write_current(alice, true, false, true);
        shell.write_current(bob, true, false, true);
        shell.enqueue(tile(TEAM, COHORT, 113), 5);

        shell.tick().await;
        let deleted_key = Stage2Key {
            partition_id: PARTITION,
            team_id: TEAM as u64,
            cohort_id: COHORT as u64,
            person_id: alice,
        };
        shell
            .store
            .write_batch(|batch| batch.delete_stage2(&deleted_key))
            .unwrap();

        shell.tick().await;
        shell.tick().await;
        shell.tick().await;

        assert_eq!(
            sink.changes()
                .iter()
                .map(|change| change.person_id.clone())
                .collect::<Vec<_>>(),
            vec![alice.to_string(), bob.to_string()],
            "the tombstoned scan row is not evaluated or emitted from its dirty marker",
        );
        assert!(shell.store.get_stage2(&deleted_key).unwrap().is_none());
        assert_eq!(shell.markers.markers().len(), 1);
    }

    #[tokio::test]
    async fn single_leaf_fix_never_cascades() {
        let sink = CaptureSink::new();
        let cascades = CaptureCascadeSink::new();
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), cascades.clone(), 8);
        let alice = Uuid::from_u128(1);
        shell.write_current(alice, true, false, false);
        shell.enqueue(tile(TEAM, COHORT, 12), 5);

        shell.tick().await;

        assert_eq!(sink.changes().len(), 1);
        assert_eq!(sink.changes()[0].status, MembershipStatus::Entered);
        assert!(shell.stage2_bit(alice));
        assert!(cascades.messages().is_empty());
        shell.tick().await;
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn a_partial_slice_withholds_without_rows_and_the_next_run_still_completes() {
        let sink = CaptureSink::new();
        let mut shell =
            DrainShell::new(false, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
        shell.cover(SliceCoverage::Since(CoverageStartMs(SLICE_START)));
        shell.fail_first_marker();
        shell.write_current(Uuid::from_u128(1), true, true, true);
        let before_slice =
            |run_id| bounded_tile(TEAM, COHORT, run_id, ScopeKind::Behavioral, SLICE_START - 1);
        shell.enqueue(before_slice(1), 5);
        shell.enqueue(before_slice(3), 6);
        shell.enqueue(tile(TEAM, COHORT, 2), 7);
        // Not part of the head's withheld burst.
        shell.enqueue(before_slice(4), 8);

        // The produce fails, so nothing is released or scanned.
        shell.tick().await;
        assert!(shell.markers.markers().is_empty());
        assert!(sink.changes().is_empty());
        assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(1))));
        assert_eq!(shell.committable(), Some(5));

        // One produce for both withheld jobs.
        shell.tick().await;
        assert_eq!(shell.markers.produce_calls(), 2, "one produce per tick");
        assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(2))));
        assert_eq!(shell.committable(), Some(7));

        for _ in 0..8 {
            if shell.queue.front().is_none() {
                break;
            }
            shell.tick().await;
        }

        let partial = WithheldReason::PartialCoverage {
            covered_since: CoverageStartMs(SLICE_START),
        };
        let markers = shell.markers.markers();
        assert!(
            matches!(
                markers.as_slice(),
                [
                    ReconcileMarker::Withheld(first),
                    ReconcileMarker::Withheld(second),
                    ReconcileMarker::Complete(complete),
                    ReconcileMarker::Withheld(last),
                ] if first.run_id() == RunId(Uuid::from_u128(1))
                    && second.run_id() == RunId(Uuid::from_u128(3))
                    && complete.run_id() == RunId(Uuid::from_u128(2))
                    && last.run_id() == RunId(Uuid::from_u128(4))
                    && [first, second, last].iter().all(|marker| marker.reason() == partial)
            ),
            "{markers:?}",
        );
        assert!(
            sink.changes()
                .iter()
                .all(|change| change.run_id == Some(RunId(Uuid::from_u128(2)))),
            "a slice missing the run's history emits no rows for it",
        );
        assert!(!sink.changes().is_empty());
        assert!(shell.deps.reconcile.backlog.is_empty());
        assert_eq!(shell.committable(), Some(9));
    }

    #[tokio::test]
    async fn a_withheld_burst_produces_at_most_one_scan_page_per_tick() {
        let mut shell = DrainShell::new(
            false,
            Arc::new(CaptureSink::new()),
            CaptureCascadeSink::new(),
            2,
        );
        shell.cover(SliceCoverage::Since(CoverageStartMs(SLICE_START)));
        let before_slice =
            |run_id| bounded_tile(TEAM, COHORT, run_id, ScopeKind::Behavioral, SLICE_START - 1);
        for run_id in 1..=3 {
            shell.enqueue(before_slice(run_id), 4 + run_id as i64);
        }

        shell.tick().await;

        assert_eq!(shell.markers.markers().len(), 2);
        assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(3))));
    }

    /// Fails the first marker of its first produce and acks the rest.
    #[derive(Default)]
    struct FailHeadOnce(CaptureReconcileMarkerSink, AtomicBool);

    #[async_trait]
    impl ReconcileMarkerSink for FailHeadOnce {
        async fn produce(
            &self,
            markers: Vec<ReconcileMarker>,
        ) -> Vec<Result<(), KafkaProduceError>> {
            if self.1.swap(true, Ordering::SeqCst) {
                return self.0.produce(markers).await;
            }
            let mut acks = vec![Err(KafkaProduceError::KafkaProduceCanceled)];
            acks.extend(self.0.produce(markers[1..].to_vec()).await);
            acks
        }
    }

    #[tokio::test]
    async fn a_later_ack_never_releases_an_unacked_withheld_head() {
        let mut shell = DrainShell::new(
            false,
            Arc::new(CaptureSink::new()),
            CaptureCascadeSink::new(),
            8,
        );
        shell.cover(SliceCoverage::Since(CoverageStartMs(SLICE_START)));
        shell.deps.reconcile.marker_sink = Arc::new(FailHeadOnce::default());
        let before_slice =
            |run_id| bounded_tile(TEAM, COHORT, run_id, ScopeKind::Behavioral, SLICE_START - 1);
        shell.enqueue(before_slice(1), 5);
        shell.enqueue(before_slice(2), 6);

        shell.tick().await;

        assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(1))));
        assert_eq!(
            shell.committable(),
            Some(5),
            "a later ack must not release an unacked head"
        );
    }

    #[tokio::test]
    async fn membership_failure_retries_the_same_page_without_advancing() {
        let sink = CaptureSink::failing_first(1);
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
        let alice = Uuid::from_u128(1);
        shell.write_current(alice, true, false, false);
        shell.enqueue(tile(TEAM, COHORT, 13), 5);

        shell.tick().await;

        assert!(sink.changes().is_empty());
        assert!(shell.markers.markers().is_empty());
        assert!(!shell.stage2_bit(alice));
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::Scanning { cursor: None }),
        ));

        shell.tick().await;

        assert_eq!(sink.changes().len(), 1);
        assert!(shell.stage2_bit(alice));
        shell.tick().await;
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(shell.committable(), Some(6));
    }

    /// A page wider than one read section still settles as one unit: the produce sees the whole
    /// page, and a failure re-reads every row rather than resuming from the sections already read.
    #[tokio::test]
    async fn a_page_spanning_read_sections_retries_as_one_unit_after_a_membership_failure() {
        let sink = CaptureSink::failing_first(1);
        let rows = MAX_ROWS + 1;
        let mut shell =
            DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 64);
        let people: Vec<Uuid> = (1..=rows as u128).map(Uuid::from_u128).collect();
        for &who in &people {
            shell.write_current(who, true, false, false);
        }
        shell.enqueue(tile(TEAM, COHORT, 16), 5);

        shell.tick().await;

        assert_eq!(sink.produce_calls(), 1, "one produce for the whole page");
        assert!(sink.changes().is_empty());
        assert!(shell.markers.markers().is_empty());
        assert!(
            people.iter().all(|&who| !shell.stage2_bit(who)),
            "no row of a failed page commits",
        );
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::Scanning { cursor: None }),
        ));
        assert_eq!(shell.committable(), Some(5));

        shell.tick().await;

        assert_eq!(
            sink.changes().len(),
            rows,
            "the whole page was emitted again"
        );
        assert!(people.iter().all(|&who| shell.stage2_bit(who)));
        shell.tick().await;
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn cascade_failure_retries_before_committing_or_advancing() {
        let sink = CaptureSink::new();
        let cascades = CaptureCascadeSink::failing_first(1);
        let mut shell = DrainShell::new(false, Arc::new(sink.clone()), cascades.clone(), 8);
        let alice = Uuid::from_u128(1);
        shell.write_current(alice, true, true, false);
        shell.enqueue(tile(TEAM, COHORT, 14), 5);

        shell.tick().await;

        assert_eq!(sink.changes().len(), 1, "the membership leg acked");
        assert!(shell.markers.markers().is_empty());
        assert!(!shell.stage2_bit(alice));
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::Scanning { cursor: None }),
        ));

        shell.tick().await;

        assert_eq!(sink.changes().len(), 2, "the page was emitted again");
        assert_eq!(cascades.messages().len(), 1);
        assert!(shell.stage2_bit(alice));
        shell.tick().await;
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn marker_failure_retries_only_the_marker_phase() {
        let sink = CaptureSink::new();
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
        shell.fail_first_marker();
        let alice = Uuid::from_u128(1);
        shell.write_current(alice, true, false, false);
        shell.enqueue(tile(TEAM, COHORT, 15), 5);

        shell.tick().await;
        shell.tick().await;

        assert_eq!(sink.changes().len(), 1);
        assert!(shell.markers.markers().is_empty());
        assert!(shell.stage2_bit(alice), "the page committed before marker");
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::MarkerReady),
        ));
        assert_eq!(shell.committable(), Some(5));

        shell.tick().await;

        assert_eq!(sink.changes().len(), 1, "the page was not rescanned");
        assert_eq!(shell.markers.markers().len(), 1);
        assert!(shell.queue.front().is_none());
        assert_eq!(shell.committable(), Some(6));
    }

    #[tokio::test]
    async fn marker_retry_drains_new_dirty_work_without_rescanning_clean_rows() {
        let sink = CaptureSink::new();
        let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
        shell.fail_first_marker();
        let alice = Uuid::from_u128(2);
        let bob = Uuid::from_u128(1);
        shell.write_current(alice, true, false, true);
        shell.enqueue(tile(TEAM, COHORT, 16), 5);

        shell.tick().await;
        shell.tick().await;
        assert_eq!(sink.changes().len(), 1);
        assert!(shell.markers.markers().is_empty());
        assert!(matches!(
            shell.queue.front().map(|job| &job.phase),
            Some(ScanPhase::MarkerReady),
        ));

        shell.write_current(bob, true, false, true);
        shell.tick().await;

        assert_eq!(
            sink.changes()
                .iter()
                .map(|change| change.person_id.clone())
                .collect::<Vec<_>>(),
            vec![alice.to_string(), bob.to_string()],
            "a failed marker drains only the person that changed while the marker was pending",
        );
        assert_eq!(shell.markers.markers().len(), 1);
        assert_eq!(shell.committable(), Some(6));
    }
}