Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

81.0 KB
Raw Blame
//! PostgreSQL contract for the seeder store, split into independent scenarios.
//!
//! Each `#[tokio::test]` connects its own schema-scoped database ([`support::with_db`]) and builds
//! only the minimal run/chunk state its invariant needs; setup duplication across scenarios is
//! deliberate and keeps them parallel-safe. Together they cover the same claim/lease/epoch-fence/CAS
//! guarantees the former single `exercise_contract` did — every assertion preserved, redistributed.

#![cfg(feature = "pg-test-support")]

use std::num::NonZeroU16;
use std::time::Duration;

use anyhow::{bail, ensure, Context, Result};
use chrono::{DateTime, Utc};
use cohort_core::filters::{CohortId, TeamId};
use cohort_seeder::app::reconcile_dispatch::{
    prepare_reconcile_dispatch, CompletionRequirement, PrepareReconcileDispatchError,
    RegisterBackfillConfirmation,
};
use cohort_seeder::app::{eligible_run_ids, fail_exhausted_runs_of_kind};
use cohort_seeder::domain::{
    tile_ranges, AttemptCount, ClaimCandidate, ClaimCandidates, ClaimEpoch, PersonRunValidation,
    PinnedWarning, ProduceHwms, RetryBackoffPolicy, RunId, ScanVolume, ScopeKind, SlotLoad,
};
use cohort_seeder::store::chunks::{ChunkStoreError, PgChunkStore, PlanOutcome, NO_ERROR_RECORDED};
use cohort_seeder::store::lease::LeaseFailure;
use cohort_seeder::store::runs::{
    complete_trailing_runs, discover_runs, establish_boundary, fail_run, load_reconcile_run,
    record_run_warning, BoundaryOutcome, ReconcileRunError, RunError, RunKind, RunStatus,
    RunWarningNote, SeedPhase,
};
use cohort_seeder::store::{Claimant, LeaseDuration, MaxAttempts, RenderedError};
use cohort_seeder::test_support;
use common_types::cohort::TeamAllowlist;
use serde_json::json;
use uuid::Uuid;

mod support;
use support::{
    behavioral_filter, empty_pinned, ensure_lease_lost, historical, insert_access,
    insert_participation, insert_person_run, insert_run, person_filter, person_pinned,
    pinned_condition, planned_count, trailing, with_db, ACTIVE_HASH, SUPERSEDED_HASH,
};

const ONE_BAND: NonZeroU16 = NonZeroU16::MIN;
/// Fail a chunk without holding it out of the claim gate, for the scenarios that assert on
/// something other than the backoff and want the chunk claimable on the next call.
const NO_BACKOFF: Duration = Duration::ZERO;

/// Candidates that hold no claim slot, so they claim in kind, day and band order.
fn idle(run_ids: &[RunId]) -> ClaimCandidates {
    let candidates = run_ids
        .iter()
        .map(|&run_id| ClaimCandidate {
            run_id,
            load: SlotLoad::default(),
        })
        .collect();
    ClaimCandidates::new(candidates).expect("every scenario claims from at least one run")
}

/// Discovery honors an `Only` allowlist plus the teams the access record has granted for 120 s, and
/// `All` admits every eligible run regardless of team, trigger, or already-seeding status.
#[tokio::test]
async fn discovery_scopes_to_the_allowlist_and_all_admits_every_run() -> Result<()> {
    with_db(|pool| async move {
        let self_run = insert_run(
            &pool,
            2,
            "team_enablement",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;
        let other_team_run = insert_run(
            &pool,
            3,
            "team_enablement",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;
        let dr_run = insert_run(
            &pool,
            4,
            "disaster_recovery",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;
        let cross_team_run =
            insert_run(&pool, 5, "team_enablement", "seeding", true, empty_pinned()).await?;
        // Grant ages sit 30 s either side of the 120 s wait, so a changed wait fails this test.
        for (team_id, level, granted_ago) in [
            (6, "processing", Duration::from_secs(150)),
            (7, "serving", Duration::from_secs(150)),
            (8, "processing", Duration::from_secs(90)),
            (9, "off", Duration::from_secs(150)),
        ] {
            insert_access(&pool, team_id, level, granted_ago).await?;
            insert_run(
                &pool,
                team_id,
                "team_enablement",
                "awaiting_boundary",
                false,
                empty_pinned(),
            )
            .await?;
        }

        let only_team_two: TeamAllowlist = "2".parse().map_err(anyhow::Error::msg)?;
        let discovered = discover_runs(&pool, &only_team_two, &[RunKind::Behavioral]).await?;
        let mut discovered_teams = discovered
            .iter()
            .map(|run| run.team_id.0)
            .collect::<Vec<_>>();
        discovered_teams.sort_unstable();
        ensure!(discovered_teams == [2, 6, 7], "{discovered_teams:?}");

        let all_runs = discover_runs(&pool, &TeamAllowlist::All, &[RunKind::Behavioral]).await?;
        ensure!(all_runs.iter().any(|run| run.run_id == self_run));
        ensure!(all_runs.iter().any(|run| run.run_id == other_team_run));
        ensure!(all_runs.iter().any(|run| run.run_id == cross_team_run));
        ensure!(all_runs.iter().any(|run| run.run_id == dr_run));
        Ok(())
    })
    .await
}

/// Two racing `establish_boundary` calls on the same awaiting-boundary run: the CAS admits exactly
/// one `Established`, and the loser re-reads the now-seeding row as `AlreadyEstablished`.
#[tokio::test]
async fn concurrent_boundary_establishment_promotes_exactly_one_run() -> Result<()> {
    with_db(|pool| async move {
        insert_run(
            &pool,
            2,
            "team_enablement",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;

        let only_team_two: TeamAllowlist = "2".parse().map_err(anyhow::Error::msg)?;
        let discovered = discover_runs(&pool, &only_team_two, &[RunKind::Behavioral]).await?;
        let left = discovered[0].clone();
        let right = discovered[0].clone();
        let (left, right) = tokio::join!(
            establish_boundary(&pool, left),
            establish_boundary(&pool, right)
        );
        let outcomes = [left?, right?];

        ensure!(
            outcomes
                .iter()
                .filter(|outcome| matches!(outcome, BoundaryOutcome::Established(_)))
                .count()
                == 1
        );
        ensure!(
            outcomes
                .iter()
                .filter(|outcome| matches!(outcome, BoundaryOutcome::AlreadyEstablished(_)))
                .count()
                == 1
        );
        Ok(())
    })
    .await
}

/// Warning notes append newest-first, dedupe on repeat, truncate the `error` column to the limit,
/// and are refused once a run leaves the `seeding` state.
#[tokio::test]
async fn run_warnings_dedupe_append_truncate_and_skip_terminal_runs() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        sqlx::query("UPDATE cohort_backfill_runs SET error = repeat('x', 5000) WHERE id = $1")
            .bind(seeding_run)
            .execute(&pool)
            .await?;

        ensure!(record_run_warning(&pool, seeding_run, RunWarningNote::ConditionsDropped).await?);
        ensure!(!record_run_warning(&pool, seeding_run, RunWarningNote::ConditionsDropped).await?);
        ensure!(record_run_warning(&pool, seeding_run, RunWarningNote::LookbackTruncated).await?);
        let run_warnings: String =
            sqlx::query_scalar("SELECT error FROM cohort_backfill_runs WHERE id = $1")
                .bind(seeding_run)
                .fetch_one(&pool)
                .await?;
        ensure!(run_warnings.len() == 4_096);
        ensure!(
            run_warnings
                .matches(RunWarningNote::ConditionsDropped.as_str())
                .count()
                == 1
        );
        ensure!(
            run_warnings
                .matches(RunWarningNote::LookbackTruncated.as_str())
                .count()
                == 1
        );

        let terminal_run =
            insert_run(&pool, 3, "team_enablement", "failed", false, empty_pinned()).await?;
        ensure!(!record_run_warning(&pool, terminal_run, RunWarningNote::ConditionsDropped).await?);
        Ok(())
    })
    .await
}

/// Loading the pinned payload drops superseded conditions (keeping only the live one plus a
/// `ConditionSuperseded` warning), rejects cohorts stored under another team, and refuses a
/// disaster-recovery run whose boundary was never pinned.
#[tokio::test]
async fn load_pinned_drops_superseded_and_rejects_cross_team_and_dr() -> Result<()> {
    with_db(|pool| async move {
        let pinned = json!({
            "schema_version": 1,
            "conditions": [
                pinned_condition(10, ACTIVE_HASH, "active-event"),
                pinned_condition(11, SUPERSEDED_HASH, "superseded-event"),
            ],
            "event_names": ["active-event", "superseded-event"],
        });
        let self_run = insert_run(
            &pool,
            2,
            "team_enablement",
            "awaiting_boundary",
            false,
            pinned,
        )
        .await?;
        insert_participation(
            &pool,
            self_run,
            2,
            10,
            false,
            behavioral_filter(ACTIVE_HASH, "active-event"),
        )
        .await?;
        insert_participation(
            &pool,
            self_run,
            2,
            11,
            true,
            behavioral_filter(SUPERSEDED_HASH, "superseded-event"),
        )
        .await?;

        let dr_run = insert_run(
            &pool,
            4,
            "disaster_recovery",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;
        let cross_team_run =
            insert_run(&pool, 5, "team_enablement", "seeding", true, empty_pinned()).await?;
        insert_participation(
            &pool,
            cross_team_run,
            999,
            50,
            true,
            json!({"properties": {"type": "AND", "values": []}}),
        )
        .await?;

        let all_runs = discover_runs(&pool, &TeamAllowlist::All, &[RunKind::Behavioral]).await?;
        let self_discovered = all_runs
            .iter()
            .find(|run| run.run_id == self_run)
            .cloned()
            .context("self run was not discovered")?;
        let cross_team_discovered = all_runs
            .iter()
            .find(|run| run.run_id == cross_team_run)
            .cloned()
            .context("cross-team run was not discovered")?;
        let dr_discovered = all_runs
            .into_iter()
            .find(|run| run.run_id == dr_run)
            .context("DR run was not discovered")?;

        let seedable = match establish_boundary(&pool, self_discovered).await? {
            BoundaryOutcome::Established(run) | BoundaryOutcome::AlreadyEstablished(run) => run,
            BoundaryOutcome::NoLongerSeedable { .. } => bail!("self run was not seedable"),
        };
        let validated = seedable.load_pinned(&pool).await?;
        ensure!(validated.run.conditions.len() == 1);
        ensure!(validated.run.conditions[0].cohort_id == CohortId(10));
        ensure!(
            validated
                .run
                .event_names
                .iter()
                .map(String::as_str)
                .collect::<Vec<_>>()
                == ["active-event"]
        );
        ensure!(validated
            .run
            .filters
            .behavioral_by_event_name
            .contains_key("active-event"));
        ensure!(!validated
            .run
            .filters
            .behavioral_by_event_name
            .contains_key("superseded-event"));
        ensure!(validated.warnings.iter().any(|warning| matches!(
            warning,
            PinnedWarning::ConditionSuperseded { cohort_id, .. } if *cohort_id == CohortId(11)
        )));

        // Already seeding with a boundary, so it promotes to a SeedableRun without a DB write.
        let cross_team_seedable = match establish_boundary(&pool, cross_team_discovered).await? {
            BoundaryOutcome::Established(run) | BoundaryOutcome::AlreadyEstablished(run) => run,
            BoundaryOutcome::NoLongerSeedable { .. } => bail!("cross-team run was not seedable"),
        };
        ensure!(matches!(
            cross_team_seedable.load_pinned(&pool).await,
            Err(RunError::CrossTeamParticipation {
                actual_team_id: 999,
                ..
            })
        ));
        ensure!(matches!(
            establish_boundary(&pool, dr_discovered).await,
            Err(RunError::DisasterRecoveryBoundaryMissing(id)) if id == dr_run
        ));
        Ok(())
    })
    .await
}

/// Manual dispatch reads only active participations, carries the behavioral-only shape hash onto
/// the control tile, and refuses stale run states or malformed tenant/hash data before Kafka.
#[tokio::test]
async fn reconcile_run_load_is_fail_closed_and_behavioral_hash_scoped() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_run(
            &pool,
            20,
            "team_enablement",
            "seeding",
            true,
            empty_pinned(),
        )
        .await?;
        insert_participation(
            &pool,
            run_id,
            20,
            201,
            false,
            behavioral_filter(ACTIVE_HASH, "active-event"),
        )
        .await?;
        insert_participation(
            &pool,
            run_id,
            20,
            202,
            true,
            behavioral_filter(SUPERSEDED_HASH, "superseded-event"),
        )
        .await?;

        let register_backfill = RegisterBackfillConfirmation::confirmed_by_operator();
        // A seeding run that never planned has no completion proof, so a complete dispatch fails
        // closed (the CLI must use --allow-incomplete for an unplanned run).
        ensure!(matches!(
            prepare_reconcile_dispatch(
                &pool,
                run_id,
                CompletionRequirement::Complete,
                register_backfill,
            )
            .await,
            Err(PrepareReconcileDispatchError::PlanningUnproven(id)) if id == run_id
        ));
        let overridden = prepare_reconcile_dispatch(
            &pool,
            run_id,
            CompletionRequirement::AllowIncomplete,
            register_backfill,
        )
        .await?;
        ensure!(overridden.total_chunks() == 0);
        ensure!(overridden.remaining_chunks() == 0);

        let reconcile = load_reconcile_run(&pool, run_id).await?;
        ensure!(reconcile.run_id() == run_id);
        ensure!(reconcile.status() == RunStatus::Seeding);
        ensure!(reconcile.cohort_count() == 1);
        let tile = reconcile
            .tiles()
            .next()
            .context("missing active cohort tile")?;
        ensure!(tile.team_id() == TeamId(20));
        ensure!(tile.cohort_id() == CohortId(201));
        ensure!(tile.scope().hash_str() == "behavioral-shape");
        ensure!(tile.scope().kind() == ScopeKind::Behavioral);
        ensure!(tile.run_id() == run_id);

        sqlx::query("UPDATE cohort_backfill_runs SET status = 'reconciling' WHERE id = $1")
            .bind(run_id)
            .execute(&pool)
            .await?;
        ensure!(load_reconcile_run(&pool, run_id).await?.status() == RunStatus::Reconciling);
        // Reaching `reconciling` is not itself a planning proof — only the stamp is.
        ensure!(matches!(
            prepare_reconcile_dispatch(
                &pool,
                run_id,
                CompletionRequirement::Complete,
                register_backfill,
            )
            .await,
            Err(PrepareReconcileDispatchError::PlanningUnproven(id)) if id == run_id
        ));

        sqlx::query("UPDATE cohort_backfill_runs SET chunks_planned_at = now() WHERE id = $1")
            .bind(run_id)
            .execute(&pool)
            .await?;
        ensure!(prepare_reconcile_dispatch(
            &pool,
            run_id,
            CompletionRequirement::Complete,
            register_backfill,
        )
        .await
        .is_ok());

        sqlx::query(
            "UPDATE cohort_backfill_run_cohorts SET behavioral_filters_shape_hash = '' \
             WHERE run_id = $1 AND cohort_id = 201",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        ensure!(matches!(
            load_reconcile_run(&pool, run_id).await,
            Err(ReconcileRunError::InvalidShapeHash { cohort_id, kind, .. })
                if cohort_id == CohortId(201) && kind == ScopeKind::Behavioral
        ));

        sqlx::query(
            "UPDATE cohort_backfill_run_cohorts \
             SET behavioral_filters_shape_hash = 'behavioral-shape', team_id = 999 \
             WHERE run_id = $1 AND cohort_id = 201",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        ensure!(matches!(
            load_reconcile_run(&pool, run_id).await,
            Err(ReconcileRunError::CrossTeamParticipation {
                expected_team_id: 20,
                actual_team_id: 999,
                ..
            })
        ));

        sqlx::query(
            "UPDATE cohort_backfill_run_cohorts SET superseded_at = now() WHERE run_id = $1",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        ensure!(matches!(
            load_reconcile_run(&pool, run_id).await,
            Err(ReconcileRunError::NoActiveParticipations(id)) if id == run_id
        ));

        let completed_run = insert_run(
            &pool,
            21,
            "team_enablement",
            "completed",
            true,
            empty_pinned(),
        )
        .await?;
        insert_participation(
            &pool,
            completed_run,
            21,
            211,
            false,
            behavioral_filter(ACTIVE_HASH, "active-event"),
        )
        .await?;
        ensure!(matches!(
            load_reconcile_run(&pool, completed_run).await,
            Err(ReconcileRunError::Status {
                status: RunStatus::Completed,
                ..
            })
        ));
        Ok(())
    })
    .await
}

/// Planning a seeding run's days is idempotent (re-planning inserts nothing), stamps the run's team
/// onto every chunk, and reports `RunNotSeeding` for a run that is not seeding.
#[tokio::test]
async fn planning_is_idempotent_scopes_team_and_gates_on_seeding() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());

        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100, 101]), ONE_BAND)
                    .await?
            )? == 2
        );
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100, 101]), ONE_BAND)
                    .await?
            )? == 0
        );
        let progress = store.chunk_progress(seeding_run).await?;
        ensure!(progress.total() == 2);
        ensure!(progress.remaining() == 2);
        let chunk_teams_match: bool = sqlx::query_scalar(
            "SELECT bool_and(team_id = 2) FROM cohort_backfill_chunks WHERE run_id = $1",
        )
        .bind(seeding_run)
        .fetch_one(&pool)
        .await?;
        ensure!(chunk_teams_match);

        let idle_run = insert_run(
            &pool,
            3,
            "team_enablement",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;
        ensure!(matches!(
            store
                .plan_chunks(idle_run, historical([200]), ONE_BAND)
                .await?,
            PlanOutcome::RunNotSeeding
        ));
        Ok(())
    })
    .await
}

/// A multi-band plan fans a day into one chunk per band (idempotently), and a claim decodes the
/// band with the day's full band count so the scan predicate partitions persons correctly.
#[tokio::test]
async fn planning_fans_out_bands_and_claims_carry_the_band_count() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let four_bands = NonZeroU16::new(4).context("four is non-zero")?;
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100]), four_bands)
                    .await?
            )? == 4
        );
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100]), four_bands)
                    .await?
            )? == 0
        );

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let claimed = store
            .claim_next(
                &idle(&[seeding_run]),
                &Claimant::new("worker-a")?,
                lease60,
                attempts5,
            )
            .await?
            .context("claimant found no chunk")?;
        let band = claimed.chunk.spec().band;
        ensure!(band.num_bands().get() == 4);
        ensure!(band.band() < 4);
        drop(claimed);
        Ok(())
    })
    .await
}

/// Two racing claimants take disjoint chunks — `SKIP LOCKED` never hands the same chunk to both.
#[tokio::test]
async fn concurrent_claims_take_disjoint_chunks() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100, 101]), ONE_BAND)
                    .await?
            )? == 2
        );

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let candidates = idle(&[seeding_run]);
        let first_store = store.clone();
        let second_store = store.clone();
        let claimant_a = Claimant::new("worker-a")?;
        let claimant_b = Claimant::new("worker-b")?;
        let (first, second) = tokio::join!(
            first_store.claim_next(&candidates, &claimant_a, lease60, attempts5),
            second_store.claim_next(&candidates, &claimant_b, lease60, attempts5),
        );
        let first = first?.context("first claimant found no chunk")?;
        let second = second?.context("second claimant found no chunk")?;
        ensure!(first.chunk.spec().lease.chunk_id() != second.chunk.spec().lease.chunk_id());
        Ok(())
    })
    .await
}

/// An expired lease is reclaimed as the SAME chunk with a bumped epoch; the reclaim fences the stale
/// lease so every one of its fenced operations reports `LeaseLost`, while the fresh lease works.
#[tokio::test]
async fn expired_lease_reclaim_bumps_epoch_and_fences_the_stale_lease() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];

        let first = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
            .await?
            .context("first claimant found no chunk")?;
        let stale = first.chunk.spec().lease;
        drop(first);
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET lease_expires_at = now() - interval '1 second' WHERE id = $1",
        )
        .bind(stale.chunk_id())
        .execute(&pool)
        .await?;

        let reclaimed = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
            .await?
            .context("expired chunk was not reclaimed")?;
        let reclaimed_lease = reclaimed.chunk.spec().lease;
        ensure!(reclaimed_lease.chunk_id() == stale.chunk_id());
        ensure!(reclaimed_lease.epoch() == ClaimEpoch(stale.epoch().0 + 1));

        ensure_lease_lost(
            test_support::heartbeat(&store, stale, &Claimant::new("worker-a")?, lease60).await,
        )?;
        ensure_lease_lost(test_support::mark_produced_raw(&store, stale, 1, ScanVolume::default()).await)?;
        ensure_lease_lost(test_support::confirm_raw(&store, stale, &ProduceHwms::default()).await)?;
        ensure_lease_lost(test_support::fail(&store, stale, "stale failure", NO_BACKOFF).await)?;
        ensure_lease_lost(test_support::unclaim(&store, stale).await)?;

        // The fresh lease, in contrast, drives the chunk through mark-produced and confirm.
        let mut hwms = ProduceHwms::default();
        hwms.observe(3, 41);
        test_support::mark_produced_raw(&store, reclaimed_lease, 0, ScanVolume::default()).await?;
        test_support::confirm_raw(&store, reclaimed_lease, &hwms).await?;
        drop(reclaimed);
        Ok(())
    })
    .await
}

/// Unclaiming a scanning chunk returns it to `pending` and refunds the attempt the claim charged.
#[tokio::test]
async fn unclaim_returns_chunk_to_pending_and_refunds_one_attempt() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100]), ONE_BAND)
                    .await?
            )? == 1
        );

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];

        let claimed = store
            .claim_next(
                &idle(&run_ids),
                &Claimant::new("worker-a")?,
                lease60,
                attempts5,
            )
            .await?
            .context("claimant found no chunk")?;
        let lease = claimed.chunk.spec().lease;
        let chunk_id = lease.chunk_id();
        sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 5 WHERE id = $1")
            .bind(chunk_id)
            .execute(&pool)
            .await?;
        store.unclaim(lease).await?;
        drop(claimed);

        let (pending_status, refunded_attempts): (String, i32) =
            sqlx::query_as("SELECT status, attempts FROM cohort_backfill_chunks WHERE id = $1")
                .bind(chunk_id)
                .fetch_one(&pool)
                .await?;
        ensure!(pending_status == "pending");
        ensure!(refunded_attempts == 4);
        Ok(())
    })
    .await
}

/// A failed chunk is held out of the claim gate until the wait `fail` stamped has elapsed, then
/// becomes claimable again with the stamp cleared. Without the hold, the poll loop re-claims a
/// chunk failing for a durable reason within seconds and burns its whole attempt budget on the same
/// failure, which is the reclaim storm this gate exists to break.
#[tokio::test]
async fn a_failed_chunk_waits_out_its_backoff_before_it_is_claimable_again() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];

        let claimed = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
            .await?
            .context("claimant found no chunk")?;
        let lease = claimed.chunk.spec().lease;
        let chunk_id = lease.chunk_id();
        // The claim reports the attempt the backoff is sized from.
        ensure!(claimed.chunk.spec().attempt.get() == 1);
        store
            .fail(
                lease,
                &RenderedError::from_message("transient"),
                Duration::from_secs(600),
            )
            .await?;
        drop(claimed);

        // The stamp is `now() + delay`, so the remaining wait is the delay minus test runtime.
        let remaining_secs: f64 = sqlx::query_scalar(
            "SELECT extract(epoch FROM next_attempt_at - now())::float8
             FROM cohort_backfill_chunks WHERE id = $1",
        )
        .bind(chunk_id)
        .fetch_one(&pool)
        .await?;
        ensure!(
            (540.0..=600.0).contains(&remaining_secs),
            "expected a ~600s hold, got {remaining_secs}s"
        );
        ensure!(store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
            .await?
            .is_none());

        // Once the wait has passed the chunk is claimable, and the claim clears the stamp so a
        // later failure is not gated by a stale one.
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET next_attempt_at = now() - interval '1 second' WHERE id = $1",
        )
        .bind(chunk_id)
        .execute(&pool)
        .await?;
        let retried = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
            .await?
            .context("chunk past its backoff was not claimable")?;
        ensure!(retried.chunk.spec().attempt.get() == 2);
        let stamp: Option<chrono::DateTime<chrono::Utc>> =
            sqlx::query_scalar("SELECT next_attempt_at FROM cohort_backfill_chunks WHERE id = $1")
                .bind(chunk_id)
                .fetch_one(&pool)
                .await?;
        ensure!(stamp.is_none(), "the claim left a stale backoff stamp");
        Ok(())
    })
    .await
}

/// The recovery path sizes the wait from the chunk's own attempt count, and that wait is what
/// reaches `next_attempt_at`.
///
/// The store's `fail` takes the delay as an argument, so every other test here supplies its own and
/// proves nothing about where a real one comes from. A recovery path that always asked for the
/// first attempt's wait would hold every chunk a flat `base` forever — defeating the whole feature
/// — while the policy's own unit tests and the gate test above all stayed green.
///
/// The chunks start one attempt below the cap, so the claim reports an attempt whose ceiling has
/// saturated at 1800s while a first attempt's is 1s. Full jitter draws from `[0, ceiling]`, so one
/// sample could still land low; over four chunks the chance that all four fall inside the first
/// attempt's 1s bound is about (1/1800)^4, which does not flake.
#[tokio::test]
async fn the_recovery_path_draws_its_wait_from_the_chunks_attempt_count() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts50 = MaxAttempts::new(50)?;
        let run_ids = [seeding_run];
        let policy =
            RetryBackoffPolicy::new(Duration::from_secs(1), Duration::from_secs(1800)).unwrap();

        let days = [100, 101, 102, 103];
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical(days), ONE_BAND)
                    .await?
            )? == 4
        );
        sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 48 WHERE run_id = $1")
            .bind(seeding_run)
            .execute(&pool)
            .await?;

        let mut longest_wait = Duration::ZERO;
        for day in days {
            let claimed = store
                .claim_next(
                    &idle(&run_ids),
                    &Claimant::new("worker-a")?,
                    lease60,
                    attempts50,
                )
                .await?
                .context("a chunk under the cap was not claimable")?;
            let chunk = claimed.chunk;
            let lease = claimed.lease;
            let chunk_id = chunk.spec().lease.chunk_id();
            ensure!(chunk.spec().attempt.get() == 49, "day {day}");
            let drawn =
                test_support::fail_via_recovery(&store, chunk, "clickhouse memory limit", policy)
                    .await
                    .context("the recovery path did not fail the chunk")?;
            drop(lease);

            // The stamp is `now() + delay`, so the remaining wait is the delay minus test runtime.
            let remaining_secs: f64 = sqlx::query_scalar(
                "SELECT extract(epoch FROM next_attempt_at - now())::float8
                 FROM cohort_backfill_chunks WHERE id = $1",
            )
            .bind(chunk_id)
            .fetch_one(&pool)
            .await?;
            ensure!(
                remaining_secs <= drawn.as_secs_f64(),
                "day {day} stamped {remaining_secs}s, more than the {drawn:?} the path drew"
            );
            longest_wait = longest_wait.max(drawn);
        }

        ensure!(
            longest_wait > policy.ceiling(AttemptCount::from_row(1)),
            "the longest of four near-cap waits was {longest_wait:?}, inside the first attempt's \
             ceiling — the recovery path is not reading the attempt count"
        );
        Ok(())
    })
    .await
}

/// Only the failed→claim transition reads the backoff stamp. A `pending` chunk has not failed, and
/// an expired `produced` lease must reclaim immediately because its tiles are already in Kafka and
/// the run cannot complete until it reaches `confirmed`. Gating either would stall a run on a
/// column that describes neither.
///
/// The two lease legs are a guard rail, not a live bug. The claim `UPDATE` sets `next_attempt_at`
/// to NULL in the same statement that writes `scanning`, so no reclaimable row carries a stale
/// stamp today and the arms would pass with or without their gate. Each leg stamps one by hand, so
/// a later edit that widens the gate onto a lease arm is caught here instead of stalling a run in
/// production.
#[tokio::test]
async fn the_backoff_gate_applies_only_to_the_failed_claim_arm() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];

        // A pending chunk carrying a future stamp is still claimable.
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET next_attempt_at = now() + interval '1 hour' WHERE run_id = $1",
        )
        .bind(seeding_run)
        .execute(&pool)
        .await?;
        let pending = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
            .await?
            .context("a pending chunk was gated by the backoff stamp")?;
        let pending_lease = pending.chunk.spec().lease;

        // An unclaim returns it to `pending`, and it stays claimable however the stamp reads.
        store.unclaim(pending_lease).await?;
        drop(pending);
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET next_attempt_at = now() + interval '1 hour' WHERE id = $1",
        )
        .bind(pending_lease.chunk_id())
        .execute(&pool)
        .await?;
        let reclaimed = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
            .await?
            .context("an unclaimed chunk was gated by the backoff stamp")?;

        // A produced chunk whose lease expired reclaims regardless of the stamp.
        let produced_lease = reclaimed.chunk.spec().lease;
        test_support::mark_produced_raw(&store, produced_lease, 1, ScanVolume::default()).await?;
        drop(reclaimed);
        sqlx::query(
            "UPDATE cohort_backfill_chunks
             SET lease_expires_at = now() - interval '1 second',
                 next_attempt_at = now() + interval '1 hour'
             WHERE id = $1",
        )
        .bind(produced_lease.chunk_id())
        .execute(&pool)
        .await?;
        let produced_reclaim = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
            .await?
            .context("an expired produced lease was gated by the backoff stamp")?;

        // A scanning chunk whose lease expired reclaims too. It is left `scanning` rather than
        // marked produced, which is the shape of a worker that died before it reached `fail`, so
        // nothing sized a wait for it and nothing should hold it back.
        let scanning_lease = produced_reclaim.chunk.spec().lease;
        drop(produced_reclaim);
        sqlx::query(
            "UPDATE cohort_backfill_chunks
             SET lease_expires_at = now() - interval '1 second',
                 next_attempt_at = now() + interval '1 hour'
             WHERE id = $1",
        )
        .bind(scanning_lease.chunk_id())
        .execute(&pool)
        .await?;
        ensure!(store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-d")?, lease60, attempts5)
            .await?
            .is_some());
        Ok(())
    })
    .await
}

/// A trailing day is planned with its hold and claimed only once the hold lapses and its run is
/// `trailing`. Discovery admits the trailing run once a hold lapses, prepare keeps it eligible, and
/// the run completes once that day confirms.
#[tokio::test]
async fn a_trailing_day_waits_out_its_hold_and_completes_its_trailing_run() -> Result<()> {
    with_db(|pool| async move {
        let run_id =
            insert_run(&pool, 2, "cohort_created", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let claimant = Claimant::new("worker-a")?;
        let run_ids = [run_id];
        let behavioral = [RunKind::Behavioral];
        let hold = Utc::now() + chrono::Duration::hours(1);
        let [historical_day] = historical([100]);
        ensure!(
            planned_count(
                store
                    .plan_chunks(run_id, [historical_day, trailing(101, hold)], ONE_BAND)
                    .await?
            )? == 2
        );
        let holds: Vec<Option<DateTime<Utc>>> = sqlx::query_scalar(
            "SELECT claimable_after FROM cohort_backfill_chunks WHERE run_id = $1 ORDER BY day",
        )
        .bind(run_id)
        .fetch_all(&pool)
        .await?;
        ensure!(
            holds
                == [
                    None,
                    DateTime::from_timestamp_millis(hold.timestamp_millis())
                ],
            "unexpected holds {holds:?}"
        );

        let historical_claim = store
            .claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
            .await?
            .context("the historical day was not claimable")?;
        ensure!(historical_claim.chunk.spec().day == 100);
        ensure!(
            store
                .claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
                .await?
                .is_none(),
            "the trailing day was claimed before its hold lapsed"
        );
        let historical_lease = historical_claim.chunk.spec().lease;
        test_support::mark_produced_raw(&store, historical_lease, 0, ScanVolume::default()).await?;
        test_support::confirm_raw(&store, historical_lease, &ProduceHwms::default()).await?;
        drop(historical_claim);

        // The finalizer's move once it has stamped readiness. Until a hold lapses nothing of the
        // run can be claimed, so discovery leaves it alone.
        sqlx::query("UPDATE cohort_backfill_runs SET status = 'trailing' WHERE id = $1")
            .bind(run_id)
            .execute(&pool)
            .await?;
        ensure!(
            discover_runs(&pool, &TeamAllowlist::All, &behavioral)
                .await?
                .iter()
                .all(|run| run.run_id != run_id),
            "the trailing run was discovered before any of its holds lapsed"
        );

        sqlx::query(
            "UPDATE cohort_backfill_chunks SET claimable_after = now() - interval '1 second'
             WHERE run_id = $1 AND claimable_after IS NOT NULL",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        let resumed = discover_runs(&pool, &TeamAllowlist::All, &behavioral)
            .await?
            .into_iter()
            .find(|run| run.run_id == run_id)
            .context("discovery skipped the trailing run once its hold lapsed")?;
        match establish_boundary(&pool, resumed).await? {
            BoundaryOutcome::AlreadyEstablished(run) => ensure!(run.phase == SeedPhase::Trailing),
            other => bail!("the trailing run resumed as {other:?}"),
        }
        ensure!(
            eligible_run_ids(&pool, &store, &TeamAllowlist::All, &behavioral)
                .await
                .contains(&run_id),
            "prepare did not keep the trailing run claim-eligible"
        );
        ensure!(complete_trailing_runs(&pool, &run_ids).await? == 0);

        let trailing_claim = store
            .claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
            .await?
            .context("the trailing day was not claimable after its hold lapsed")?;
        ensure!(trailing_claim.chunk.spec().day == 101);
        let trailing_lease = trailing_claim.chunk.spec().lease;
        test_support::heartbeat(&store, trailing_lease, &claimant, lease60).await?;
        test_support::mark_produced_raw(&store, trailing_lease, 0, ScanVolume::default()).await?;
        test_support::confirm_raw(&store, trailing_lease, &ProduceHwms::default()).await?;
        drop(trailing_claim);

        ensure!(complete_trailing_runs(&pool, &run_ids).await? == 1);
        let (status, finished): (String, bool) = sqlx::query_as(
            "SELECT status, finished_at IS NOT NULL FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(status == "completed" && finished);
        Ok(())
    })
    .await
}

/// While a run still seeds, a held chunk stays unclaimable even after its hold lapses, so the
/// reconciling CAS cannot cut its lease mid-scan; and once every chunk has confirmed, the trailing
/// sweep leaves the run in `seeding`, since completing it there would skip `reconciling` and the
/// readiness stamp.
#[tokio::test]
async fn a_seeding_run_neither_hands_out_a_lapsed_held_chunk_nor_completes_as_trailing(
) -> Result<()> {
    with_db(|pool| async move {
        let run_id =
            insert_run(&pool, 2, "cohort_created", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [run_id];
        let [historical_day] = historical([100]);
        let lapsed = Utc::now() - chrono::Duration::hours(1);
        store
            .plan_chunks(run_id, [historical_day, trailing(101, lapsed)], ONE_BAND)
            .await?;
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET status = 'confirmed'
             WHERE run_id = $1 AND claimable_after IS NULL",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        ensure!(
            store
                .claim_next(
                    &idle(&run_ids),
                    &Claimant::new("worker-a")?,
                    lease60,
                    attempts5
                )
                .await?
                .is_none(),
            "a held chunk was claimed while its run was still seeding"
        );

        sqlx::query("UPDATE cohort_backfill_chunks SET status = 'confirmed' WHERE run_id = $1")
            .bind(run_id)
            .execute(&pool)
            .await?;
        ensure!(complete_trailing_runs(&pool, &run_ids).await? == 0);
        let status: String =
            sqlx::query_scalar("SELECT status FROM cohort_backfill_runs WHERE id = $1")
                .bind(run_id)
                .fetch_one(&pool)
                .await?;
        ensure!(
            status == "seeding",
            "the seeding run was completed as if trailing"
        );
        Ok(())
    })
    .await
}

/// The `scanning`→`produced` CAS records what the scan moved, and planning leaves the byte columns
/// at the database-side default. The default is load-bearing: `plan_chunks` inserts an explicit
/// column list that names neither column, so a Django-only default would make every planning insert
/// violate NOT NULL during a rollout.
#[tokio::test]
async fn mark_produced_records_the_scan_byte_volume_that_planning_defaults_to_zero() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);

        let planned: (i64, i64) = sqlx::query_as(
            "SELECT scan_received_bytes, scan_decoded_bytes FROM cohort_backfill_chunks WHERE run_id = $1",
        )
        .bind(seeding_run)
        .fetch_one(&pool)
        .await?;
        ensure!(planned == (0, 0));

        let claimed = store
            .claim_next(
                &idle(&[seeding_run]),
                &Claimant::new("worker-a")?,
                LeaseDuration::new(Duration::from_secs(60))?,
                MaxAttempts::new(5)?,
            )
            .await?
            .context("claimant found no chunk")?;
        let lease = claimed.chunk.spec().lease;
        test_support::mark_produced_raw(
            &store,
            lease,
            7,
            ScanVolume::new(9_876_543_210, 41_234_567_890),
        )
        .await?;
        drop(claimed);

        let stored: (i64, i64, i64) = sqlx::query_as(
            "SELECT tiles_produced, scan_received_bytes, scan_decoded_bytes
             FROM cohort_backfill_chunks WHERE id = $1",
        )
        .bind(lease.chunk_id())
        .fetch_one(&pool)
        .await?;
        ensure!(stored == (7, 9_876_543_210, 41_234_567_890));
        Ok(())
    })
    .await
}

/// The attempt cap is terminal for a `failed` chunk (no further claim), but an expired `produced`
/// chunk sitting AT the cap is still reclaimed with a bumped epoch — its tiles are already in
/// Kafka, so it must keep retrying until it reaches `confirmed`. Only `scanning` reclaims are
/// capped (see `expired_scanning_chunk_at_the_cap_is_reaped_not_reclaimed`).
#[tokio::test]
async fn attempt_cap_is_terminal_for_failed_but_reclaims_expired_produced() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];

        // A chunk driven to the attempt cap by claiming, then failed, is not claimable again.
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
        sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 4 WHERE run_id = $1")
            .bind(seeding_run)
            .execute(&pool)
            .await?;
        let retry = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-d")?, lease60, attempts5)
            .await?
            .context("chunk under the cap was not claimable")?;
        let retry_lease = retry.chunk.spec().lease;
        let reclaimed_attempts: i32 =
            sqlx::query_scalar("SELECT attempts FROM cohort_backfill_chunks WHERE id = $1")
                .bind(retry_lease.chunk_id())
                .fetch_one(&pool)
                .await?;
        ensure!(reclaimed_attempts == 5);
        store
            .fail(retry_lease, &RenderedError::from_message("terminal"), NO_BACKOFF)
            .await?;
        drop(retry);
        ensure!(store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-terminal")?, lease60, attempts5)
            .await?
            .is_none());

        // A second chunk, marked produced then expired at the cap, IS reclaimed.
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([101]), ONE_BAND).await?)? == 1);
        let final_attempt = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-e")?, lease60, attempts5)
            .await?
            .context("second chunk was not claimable")?;
        let final_attempt_lease = final_attempt.chunk.spec().lease;
        test_support::mark_produced_raw(&store, final_attempt_lease, 1, ScanVolume::default()).await?;
        drop(final_attempt);
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET attempts = 5, lease_expires_at = now() - interval '1 second' WHERE id = $1",
        )
        .bind(final_attempt_lease.chunk_id())
        .execute(&pool)
        .await?;

        let observed = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-f")?, lease60, attempts5)
            .await?
            .context("expired produced chunk at the attempt cap was not reclaimed")?;
        let observed_lease = observed.chunk.spec().lease;
        ensure!(observed_lease.chunk_id() == final_attempt_lease.chunk_id());
        ensure!(observed_lease.epoch() == ClaimEpoch(final_attempt_lease.epoch().0 + 1));
        let active_reclaim_attempts: i32 =
            sqlx::query_scalar("SELECT attempts FROM cohort_backfill_chunks WHERE id = $1")
                .bind(observed_lease.chunk_id())
                .fetch_one(&pool)
                .await?;
        ensure!(active_reclaim_attempts == 5);
        // The claim `CASE` stops incrementing at the cap, so this is the arm where the count the
        // spec reports and the number of claims diverge. The spec must still carry the column, not
        // the claim tally: the retry backoff is sized from it.
        ensure!(observed.chunk.spec().attempt.get() == 5);
        Ok(())
    })
    .await
}

/// A `scanning` chunk whose lease expires below the attempt cap is left for reclaim, but once its
/// attempts saturate it is neither reclaimed (no hard-crash loop) nor left invisible: the reaper
/// dead-letters it to `failed` — the same terminal state a clean cap-out reaches — exactly once.
#[tokio::test]
async fn expired_scanning_chunk_at_the_cap_is_reaped_not_reclaimed() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];
        let claimed = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
            .await?
            .context("claimant found no chunk")?;
        let chunk_id = claimed.chunk.spec().lease.chunk_id();
        drop(claimed);

        // Expired below the cap: not reaped — the ordinary reclaim path still owns it.
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET lease_expires_at = now() - interval '1 second' WHERE id = $1",
        )
        .bind(chunk_id)
        .execute(&pool)
        .await?;
        ensure!(store.reap_poisoned_chunks(&run_ids, attempts5).await? == 0);

        // A hard-crashed worker's residue: still `scanning`, attempts saturated, lease expired.
        sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 5 WHERE id = $1")
            .bind(chunk_id)
            .execute(&pool)
            .await?;
        ensure!(store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
            .await?
            .is_none());

        ensure!(store.reap_poisoned_chunks(&run_ids, attempts5).await? == 1);
        let (status, last_error): (String, String) =
            sqlx::query_as("SELECT status, last_error FROM cohort_backfill_chunks WHERE id = $1")
                .bind(chunk_id)
                .fetch_one(&pool)
                .await?;
        ensure!(status == "failed");
        ensure!(last_error.contains("attempt cap"));
        // Terminal: not claimable, and a second sweep is a no-op.
        ensure!(store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
            .await?
            .is_none());
        ensure!(store.reap_poisoned_chunks(&run_ids, attempts5).await? == 0);
        Ok(())
    })
    .await
}

/// Only chunks that saturated the attempt cap count as exhausted. A `failed` chunk still under the
/// cap is reclaimable and will retry, so reporting it would fail runs that were about to recover.
#[tokio::test]
async fn runs_with_exhausted_chunks_selects_only_capped_failures() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND).await?)? == 2);
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];

        let chunk_ids: Vec<Uuid> = sqlx::query_scalar(
            "SELECT id FROM cohort_backfill_chunks WHERE run_id = $1 ORDER BY day",
        )
        .bind(seeding_run)
        .fetch_all(&pool)
        .await?;
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 4, last_error = 'still retrying' WHERE id = $1",
        )
        .bind(chunk_ids[0])
        .execute(&pool)
        .await?;
        ensure!(store
            .runs_with_exhausted_chunks(&run_ids, attempts5)
            .await?
            .is_empty());

        sqlx::query(
            "UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5, last_error = 'scan blew up' WHERE id = $1",
        )
        .bind(chunk_ids[1])
        .execute(&pool)
        .await?;
        let exhausted = store.runs_with_exhausted_chunks(&run_ids, attempts5).await?;
        ensure!(exhausted.len() == 1);
        ensure!(exhausted[0].run_id == seeding_run);
        ensure!(exhausted[0].exhausted == 1, "only the capped chunk counts");
        ensure!(exhausted[0].chunk_id == chunk_ids[1].to_string());
        ensure!(exhausted[0].last_error == "scan blew up");

        ensure!(store
            .runs_with_exhausted_chunks(&[], attempts5)
            .await?
            .is_empty());
        Ok(())
    })
    .await
}

/// The reported chunk and error come from one row. Aggregating them independently pairs the lowest
/// chunk id with some other chunk's error, sending the operator to read a row that never failed
/// that way.
#[tokio::test]
async fn exhausted_chunk_and_error_are_read_off_the_same_row() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND).await?)? == 2);
        let attempts5 = MaxAttempts::new(5)?;

        // Ordered so the lowest-id chunk carries the *higher* error text: an independent
        // `min(last_error)` would then hand back the other chunk's error.
        let mut chunk_ids: Vec<Uuid> =
            sqlx::query_scalar("SELECT id FROM cohort_backfill_chunks WHERE run_id = $1")
                .bind(seeding_run)
                .fetch_all(&pool)
                .await?;
        chunk_ids.sort();
        for (chunk_id, error) in chunk_ids.iter().zip(["zzz lowest id", "aaa highest id"]) {
            sqlx::query(
                "UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5, last_error = $2 WHERE id = $1",
            )
            .bind(chunk_id)
            .bind(error)
            .execute(&pool)
            .await?;
        }

        let exhausted = store
            .runs_with_exhausted_chunks(&[seeding_run], attempts5)
            .await?;
        ensure!(exhausted.len() == 1);
        ensure!(exhausted[0].exhausted == 2, "both capped chunks counted");
        ensure!(exhausted[0].chunk_id == chunk_ids[0].to_string());
        ensure!(exhausted[0].last_error == "zzz lowest id");

        // A capped chunk with no persisted error still has to render as something: the run error
        // interpolates this text, and an empty one leaves the operator a trailing colon.
        sqlx::query("UPDATE cohort_backfill_chunks SET last_error = '' WHERE id = $1")
            .bind(chunk_ids[0])
            .execute(&pool)
            .await?;
        let exhausted = store
            .runs_with_exhausted_chunks(&[seeding_run], attempts5)
            .await?;
        ensure!(exhausted[0].last_error == NO_ERROR_RECORDED);
        Ok(())
    })
    .await
}

/// Readiness does not wait for a trailing day, so that day's exhausted chunk fails its run only once
/// the run has stamped readiness and is `trailing`.
#[tokio::test]
async fn an_exhausted_trailing_chunk_fails_its_run_only_once_the_run_is_trailing() -> Result<()> {
    with_db(|pool| async move {
        let run_id =
            insert_run(&pool, 2, "cohort_created", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [run_id];
        store
            .plan_chunks(run_id, [trailing(101, Utc::now())], ONE_BAND)
            .await?;
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5 WHERE run_id = $1",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;

        ensure!(
            fail_exhausted_runs_of_kind(&pool, &store, &run_ids, RunKind::Behavioral, attempts5)
                .await
                == 0
        );
        sqlx::query("UPDATE cohort_backfill_runs SET status = 'trailing' WHERE id = $1")
            .bind(run_id)
            .execute(&pool)
            .await?;
        ensure!(
            fail_exhausted_runs_of_kind(&pool, &store, &run_ids, RunKind::Behavioral, attempts5)
                .await
                == 1
        );
        Ok(())
    })
    .await
}

/// Failing a run whose chunk exhausted its retries stops the run dead: a still-pending sibling is no
/// longer claimable, a live sibling's heartbeat is refused, and a second failure is a no-op rather
/// than a double-count. Without it the run sits in `seeding` forever holding its cohort's slot.
#[tokio::test]
async fn exhausted_chunk_fails_the_run_and_stops_further_claims() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(
            planned_count(store.plan_chunks(seeding_run, historical([100, 101, 102]), ONE_BAND).await?)? == 3
        );

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];
        let claimant = Claimant::new("worker-a")?;

        // One chunk mid-scan with a live lease, one capped out, one left pending.
        let claimed = store
            .claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
            .await?
            .context("claimant found no chunk")?;
        let live_lease = claimed.chunk.spec().lease;
        let capped_id: Uuid = sqlx::query_scalar(
            "SELECT id FROM cohort_backfill_chunks WHERE run_id = $1 AND status = 'pending' ORDER BY day LIMIT 1",
        )
        .bind(seeding_run)
        .fetch_one(&pool)
        .await?;
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5, last_error = 'scan blew up' WHERE id = $1",
        )
        .bind(capped_id)
        .execute(&pool)
        .await?;

        // Through the seeder's own pass, not a hand-rolled equivalent: the scan, the error text and
        // the attempt-cap interpolation are what an operator ends up reading.
        ensure!(
            fail_exhausted_runs_of_kind(
                &pool,
                &store,
                &run_ids,
                RunKind::Behavioral,
                attempts5
            )
            .await
                == 1
        );

        let (status, error, finished): (String, String, bool) = sqlx::query_as(
            "SELECT status, error, finished_at IS NOT NULL FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(seeding_run)
        .fetch_one(&pool)
        .await?;
        ensure!(status == "failed");
        ensure!(error.contains("1 chunk(s) exhausted the 5 attempt retry budget"));
        ensure!(error.contains(&capped_id.to_string()));
        ensure!(error.contains("scan blew up"));
        ensure!(finished);

        // The claim and heartbeat predicates both join `runs.status = 'seeding'`, so the pending
        // sibling stops being claimable and the in-flight one halts on its next beat.
        ensure!(store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
            .await?
            .is_none());
        let still_pending: i64 = sqlx::query_scalar(
            "SELECT count(*) FROM cohort_backfill_chunks WHERE run_id = $1 AND status = 'pending'",
        )
        .bind(seeding_run)
        .fetch_one(&pool)
        .await?;
        ensure!(
            still_pending == 1,
            "a claimable chunk remained, so the run status is what blocked the claim"
        );
        ensure_lease_lost(test_support::heartbeat(&store, live_lease, &claimant, lease60).await)?;
        drop(claimed);

        // Idempotent: a later pass re-reads the same exhausted chunks, and the already-terminal run
        // must not be counted again.
        ensure!(
            fail_exhausted_runs_of_kind(
                &pool,
                &store,
                &run_ids,
                RunKind::Behavioral,
                attempts5
            )
            .await
                == 0
        );
        ensure!(matches!(
            fail_run(&pool, seeding_run, &RenderedError::from_message("again")).await,
            Err(RunError::NotActive(_))
        ));
        Ok(())
    })
    .await
}

/// Both persisted error columns are truncated to the limit: `chunk.fail` clamps `last_error` and
/// `fail_run` clamps `run.error`, each flipping the row to `failed`.
#[tokio::test]
async fn fail_truncates_chunk_and_run_error_columns() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(
            planned_count(
                store
                    .plan_chunks(seeding_run, historical([100]), ONE_BAND)
                    .await?
            )? == 1
        );

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];
        let claimed = store
            .claim_next(
                &idle(&run_ids),
                &Claimant::new("worker-a")?,
                lease60,
                attempts5,
            )
            .await?
            .context("claimant found no chunk")?;
        let lease = claimed.chunk.spec().lease;
        let chunk_id = lease.chunk_id();
        store
            .fail(
                lease,
                &RenderedError::from_message("x".repeat(5_000)),
                NO_BACKOFF,
            )
            .await?;
        drop(claimed);
        let (failed_status, error_length): (String, i32) = sqlx::query_as(
            "SELECT status, length(last_error)::integer FROM cohort_backfill_chunks WHERE id = $1",
        )
        .bind(chunk_id)
        .fetch_one(&pool)
        .await?;
        ensure!(failed_status == "failed");
        ensure!(error_length == 4_096);

        let active_run = insert_run(
            &pool,
            3,
            "team_enablement",
            "awaiting_boundary",
            false,
            empty_pinned(),
        )
        .await?;
        fail_run(
            &pool,
            active_run,
            &RenderedError::from_message("y".repeat(5_000)),
        )
        .await?;
        let (run_status, run_error_length): (String, i32) = sqlx::query_as(
            "SELECT status, length(error)::integer FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(active_run)
        .fetch_one(&pool)
        .await?;
        ensure!(run_status == RunStatus::Failed.as_str());
        ensure!(run_error_length == 4_096);
        Ok(())
    })
    .await
}

/// The dark-by-default proof: discovery with `kinds = [Behavioral]` (the gate-off binding) never
/// yields a person run, while widening the kind set surfaces it with its kind and pinned
/// `person_scan_since` decoded, and the pinned load validates the person payload.
#[tokio::test]
async fn discovery_is_kind_gated_and_person_pinned_load_validates() -> Result<()> {
    with_db(|pool| async move {
        let behavioral_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let person_run = insert_person_run(
            &pool,
            2,
            "seeding",
            true,
            person_pinned(&[(10, ACTIVE_HASH), (11, SUPERSEDED_HASH)]),
        )
        .await?;
        insert_participation(
            &pool,
            person_run,
            2,
            10,
            false,
            person_filter(ACTIVE_HASH, "email"),
        )
        .await?;
        insert_participation(
            &pool,
            person_run,
            2,
            11,
            true,
            person_filter(SUPERSEDED_HASH, "plan"),
        )
        .await?;

        let behavioral_only =
            discover_runs(&pool, &TeamAllowlist::All, &[RunKind::Behavioral]).await?;
        ensure!(behavioral_only.iter().all(|run| run.run_id != person_run));
        ensure!(behavioral_only
            .iter()
            .any(|run| run.run_id == behavioral_run));

        let both_kinds = discover_runs(
            &pool,
            &TeamAllowlist::All,
            &[RunKind::Behavioral, RunKind::PersonProperty],
        )
        .await?;
        let discovered = both_kinds
            .iter()
            .find(|run| run.run_id == person_run)
            .cloned()
            .context("person run was not discovered with the widened kind set")?;
        ensure!(discovered.kind == RunKind::PersonProperty);
        ensure!(discovered.person_scan_since.is_some());
        let behavioral = both_kinds
            .iter()
            .find(|run| run.run_id == behavioral_run)
            .context("behavioral run missing from the widened discovery")?;
        ensure!(behavioral.kind == RunKind::Behavioral);

        let seedable = match establish_boundary(&pool, discovered).await? {
            BoundaryOutcome::Established(run) | BoundaryOutcome::AlreadyEstablished(run) => run,
            BoundaryOutcome::NoLongerSeedable { .. } => bail!("person run was not seedable"),
        };
        let PersonRunValidation::Seedable(validated) = seedable.load_person_pinned(&pool).await?
        else {
            bail!("person run with an active participation unexpectedly retired");
        };
        ensure!(validated.run.conditions.len() == 1);
        ensure!(validated.run.horizon_days == 30);
        ensure!(validated.warnings.iter().any(|warning| matches!(
            warning,
            PinnedWarning::ConditionSuperseded { cohort_id, .. } if *cohort_id == CohortId(11)
        )));
        Ok(())
    })
    .await
}

/// The cluster-wide planning claim is exclusive per run and frees on release, so at most one
/// replica ever runs a run's boundary scan.
#[tokio::test]
async fn person_planning_claim_is_exclusive_per_run_until_released() -> Result<()> {
    with_db(|pool| async move {
        let run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
        let other_run = insert_person_run(&pool, 3, "seeding", true, person_pinned(&[])).await?;
        let store = PgChunkStore::new(pool.clone());

        let held = store
            .try_claim_person_planning(run)
            .await?
            .context("first claim should win")?;
        ensure!(store.try_claim_person_planning(run).await?.is_none());
        // Claims are per run: a different run's planner is unaffected.
        let other = store
            .try_claim_person_planning(other_run)
            .await?
            .context("a different run's claim should win")?;

        held.release().await;
        other.release().await;
        let reclaimed = store
            .try_claim_person_planning(run)
            .await?
            .context("a released claim should be reclaimable")?;
        reclaimed.release().await;
        Ok(())
    })
    .await
}

/// Person planning is all-or-nothing: one racing planner wins and inserts every range under the
/// sentinel day with ordinal bands, the loser and any re-plan report `AlreadyPlanned`, and a
/// non-seeding run refuses.
#[tokio::test]
async fn person_planning_is_all_or_nothing_and_idempotent() -> Result<()> {
    with_db(|pool| async move {
        let run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
        let store = PgChunkStore::new(pool.clone());
        let boundaries = [Uuid::from_u128(0x10), Uuid::from_u128(0x20)];
        let ranges = tile_ranges(&boundaries)?;
        ensure!(ranges.len() == 3);

        let (left, right) = tokio::join!(
            store.plan_person_chunks(run, &ranges),
            store.plan_person_chunks(run, &ranges),
        );
        let outcomes = [left?, right?];
        ensure!(
            outcomes
                .iter()
                .filter(|outcome| matches!(outcome, PlanOutcome::Planned { inserted: 3 }))
                .count()
                == 1,
            "expected exactly one winning planner, got {outcomes:?}"
        );
        ensure!(
            outcomes
                .iter()
                .filter(|outcome| matches!(outcome, PlanOutcome::AlreadyPlanned))
                .count()
                == 1,
            "expected the losing planner to observe AlreadyPlanned, got {outcomes:?}"
        );
        ensure!(matches!(
            store.plan_person_chunks(run, &ranges).await?,
            PlanOutcome::AlreadyPlanned
        ));

        let rows: Vec<(String, i16, Option<Uuid>, Option<Uuid>)> = sqlx::query_as(
            "SELECT day::text, band, person_range_lo, person_range_hi \
             FROM cohort_backfill_chunks WHERE run_id = $1 ORDER BY band",
        )
        .bind(run)
        .fetch_all(&pool)
        .await?;
        ensure!(rows.len() == 3);
        ensure!(rows.iter().all(|(day, ..)| day == "9999-01-01"));
        ensure!(rows.iter().map(|(_, band, ..)| *band).eq(0..3));
        ensure!(rows[0].2 == Some(Uuid::nil()));
        ensure!(rows[0].3 == Some(boundaries[0]));
        ensure!(rows[1].2 == Some(boundaries[0]));
        ensure!(rows[1].3 == Some(boundaries[1]));
        ensure!(rows[2].2 == Some(boundaries[1]));
        ensure!(rows[2].3.is_none());

        let idle =
            insert_person_run(&pool, 3, "awaiting_boundary", false, person_pinned(&[])).await?;
        ensure!(matches!(
            store.plan_person_chunks(idle, &ranges).await?,
            PlanOutcome::RunNotSeeding
        ));
        Ok(())
    })
    .await
}

/// Behavioral days claim newest first across runs with equal slot loads, and person chunks claim
/// strictly after every behavioral day on the shared claim loop, even though their sentinel day is
/// the newest and their run holds fewer slots. A person claim decodes its range with `num_bands` =
/// the run's chunk count.
#[tokio::test]
async fn behavioral_days_claim_newest_first_then_person_chunks_with_ranges() -> Result<()> {
    with_db(|pool| async move {
        let behavioral_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        // Another team, because a team holds one active team-scoped run per kind.
        let other_team_run =
            insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
        let person_run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(
            planned_count(
                store
                    .plan_chunks(behavioral_run, historical([98, 100]), ONE_BAND)
                    .await?
            )? == 2
        );
        ensure!(
            planned_count(
                store
                    .plan_chunks(other_team_run, historical([99]), ONE_BAND)
                    .await?
            )? == 1
        );
        let boundary = Uuid::from_u128(0x42);
        let ranges = tile_ranges(&[boundary])?;
        ensure!(planned_count(store.plan_person_chunks(person_run, &ranges).await?)? == 2);

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let busy = SlotLoad { team: 1, run: 1 };
        let candidates = ClaimCandidates::new(vec![
            ClaimCandidate {
                run_id: behavioral_run,
                load: busy,
            },
            ClaimCandidate {
                run_id: other_team_run,
                load: busy,
            },
            ClaimCandidate {
                run_id: person_run,
                load: SlotLoad::default(),
            },
        ])
        .context("no claim candidates")?;
        let claimant = Claimant::new("worker-a")?;

        let mut behavioral_claims = Vec::new();
        for _ in 0..3 {
            let claim = store
                .claim_next(&candidates, &claimant, lease60, attempts5)
                .await?
                .context("behavioral chunk was not claimed before person chunks")?;
            ensure!(claim.chunk.spec().person_range.is_none());
            behavioral_claims.push((claim.chunk.spec().lease.run_id(), claim.chunk.spec().day));
        }
        ensure!(
            behavioral_claims
                == [
                    (behavioral_run, 100),
                    (other_team_run, 99),
                    (behavioral_run, 98)
                ],
            "expected newest day first across runs, got {behavioral_claims:?}"
        );

        let first_person = store
            .claim_next(&candidates, &claimant, lease60, attempts5)
            .await?
            .context("first person chunk was not claimed")?;
        let first_person_spec = first_person.chunk.spec();
        ensure!(first_person_spec.lease.run_id() == person_run);
        ensure!(first_person_spec.band.band() == 0);
        ensure!(first_person_spec.band.num_bands().get() == 2);
        let first_person_range = first_person_spec
            .person_range
            .context("person claim lost its range")?;
        ensure!(first_person_range.lo() == Uuid::nil());
        ensure!(first_person_range.hi() == Some(boundary));

        let second_person = store
            .claim_next(&candidates, &claimant, lease60, attempts5)
            .await?
            .context("second person chunk was not claimed")?;
        let second_person_range = second_person
            .chunk
            .spec()
            .person_range
            .context("person claim lost its range")?;
        ensure!(second_person_range.lo() == boundary);
        ensure!(second_person_range.hi().is_none());
        drop((first_person, second_person));
        Ok(())
    })
    .await
}

/// A claim takes a lapsed trailing day first, then serves the team holding the fewest slots, then
/// the run holding the fewest, and only then the newest day.
#[tokio::test]
async fn claims_serve_the_least_loaded_team_then_run_before_the_newest_day() -> Result<()> {
    with_db(|pool| async move {
        let heavy_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        // A team holds one active team-scoped run per kind, so the first run moves to cohort
        // scope before the second is inserted.
        sqlx::query(
            "UPDATE cohort_backfill_runs SET scope = 'cohort', cohort_id = 1 WHERE id = $1",
        )
        .bind(heavy_run)
        .execute(&pool)
        .await?;
        let sibling_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let other_team_run =
            insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
        let trailing_run =
            insert_run(&pool, 4, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        for (run_id, day) in [(heavy_run, 100), (sibling_run, 98), (other_team_run, 97)] {
            ensure!(
                planned_count(
                    store
                        .plan_chunks(run_id, historical([day]), ONE_BAND)
                        .await?
                )? == 1
            );
        }
        let lapsed = Utc::now() - chrono::Duration::hours(1);
        ensure!(
            planned_count(
                store
                    .plan_chunks(trailing_run, [trailing(96, lapsed)], ONE_BAND)
                    .await?
            )? == 1
        );
        sqlx::query("UPDATE cohort_backfill_runs SET status = 'trailing' WHERE id = $1")
            .bind(trailing_run)
            .execute(&pool)
            .await?;

        let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
        let attempts5 = MaxAttempts::new(5)?;
        let claimant = Claimant::new("worker-a")?;
        let rounds = [
            // Team 4 holds the most slots and its trailing day is the oldest day, but the trailing
            // day still goes first.
            ([(2, 2), (2, 0), (1, 1), (3, 0)], (trailing_run, 96)),
            // Team 2 holds two slots and team 3 one, so team 3 goes first, ahead of team 2's newer
            // days and of its sibling run that holds none.
            ([(2, 2), (2, 0), (1, 1), (4, 1)], (other_team_run, 97)),
            // Both teams hold one slot, so inside team 2 the run holding none goes first, ahead of
            // the heavy run's newer day.
            ([(1, 1), (1, 0), (1, 1), (4, 1)], (sibling_run, 98)),
        ];
        for (loads, expected) in rounds {
            let candidates = [heavy_run, sibling_run, other_team_run, trailing_run]
                .into_iter()
                .zip(loads)
                .map(|(run_id, (team, run))| ClaimCandidate {
                    run_id,
                    load: SlotLoad { team, run },
                })
                .collect();
            let candidates = ClaimCandidates::new(candidates).context("no claim candidates")?;
            let claim = store
                .claim_next(&candidates, &claimant, lease60, attempts5)
                .await?
                .context("no chunk was claimed")?;
            let claimed = (claim.chunk.spec().lease.run_id(), claim.chunk.spec().day);
            ensure!(
                claimed == expected,
                "expected {expected:?}, got {claimed:?}"
            );
        }
        Ok(())
    })
    .await
}

/// Cancelling a run breaks its live leases: the claimed chunk's next heartbeat fences out, so the
/// lease surfaces `Heartbeat(LeaseLost)` and its cancellation token trips.
#[tokio::test]
async fn cancelling_a_run_kills_its_live_lease_via_the_heartbeat() -> Result<()> {
    with_db(|pool| async move {
        let seeding_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let store = PgChunkStore::new(pool.clone());
        ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);

        let lease3 = LeaseDuration::new(Duration::from_secs(3))?;
        let attempts5 = MaxAttempts::new(5)?;
        let run_ids = [seeding_run];
        let observed = store
            .claim_next(&idle(&run_ids), &Claimant::new("worker-f")?, lease3, attempts5)
            .await?
            .context("claimant found no chunk")?;
        let cancelled = observed.lease.cancellation_token();
        sqlx::query(
            "UPDATE cohort_backfill_runs SET status = 'cancelled', updated_at = now() WHERE id = $1",
        )
        .bind(seeding_run)
        .execute(&pool)
        .await?;
        let failure =
            tokio::time::timeout(Duration::from_secs(3), observed.lease.failure()).await?;
        ensure!(matches!(
            failure,
            LeaseFailure::Heartbeat(ChunkStoreError::LeaseLost { .. })
        ));
        ensure!(cancelled.is_cancelled());
        Ok(())
    })
    .await
}