:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! PostgreSQL contract for the seeder store, split into independent scenarios.
//!
//! Each `#[tokio::test]` connects its own schema-scoped database ([`support::with_db`]) and builds
//! only the minimal run/chunk state its invariant needs; setup duplication across scenarios is
//! deliberate and keeps them parallel-safe. Together they cover the same claim/lease/epoch-fence/CAS
//! guarantees the former single `exercise_contract` did — every assertion preserved, redistributed.
#![cfg(feature = "pg-test-support")]
use std::num::NonZeroU16;
use std::time::Duration;
use anyhow::{bail, ensure, Context, Result};
use chrono::{DateTime, Utc};
use cohort_core::filters::{CohortId, TeamId};
use cohort_seeder::app::reconcile_dispatch::{
prepare_reconcile_dispatch, CompletionRequirement, PrepareReconcileDispatchError,
RegisterBackfillConfirmation,
};
use cohort_seeder::app::{eligible_run_ids, fail_exhausted_runs_of_kind};
use cohort_seeder::domain::{
tile_ranges, AttemptCount, ClaimCandidate, ClaimCandidates, ClaimEpoch, PersonRunValidation,
PinnedWarning, ProduceHwms, RetryBackoffPolicy, RunId, ScanVolume, ScopeKind, SlotLoad,
};
use cohort_seeder::store::chunks::{ChunkStoreError, PgChunkStore, PlanOutcome, NO_ERROR_RECORDED};
use cohort_seeder::store::lease::LeaseFailure;
use cohort_seeder::store::runs::{
complete_trailing_runs, discover_runs, establish_boundary, fail_run, load_reconcile_run,
record_run_warning, BoundaryOutcome, ReconcileRunError, RunError, RunKind, RunStatus,
RunWarningNote, SeedPhase,
};
use cohort_seeder::store::{Claimant, LeaseDuration, MaxAttempts, RenderedError};
use cohort_seeder::test_support;
use common_types::cohort::TeamAllowlist;
use serde_json::json;
use uuid::Uuid;
mod support;
use support::{
behavioral_filter, empty_pinned, ensure_lease_lost, historical, insert_access,
insert_participation, insert_person_run, insert_run, person_filter, person_pinned,
pinned_condition, planned_count, trailing, with_db, ACTIVE_HASH, SUPERSEDED_HASH,
};
const ONE_BAND: NonZeroU16 = NonZeroU16::MIN;
/// Fail a chunk without holding it out of the claim gate, for the scenarios that assert on
/// something other than the backoff and want the chunk claimable on the next call.
const NO_BACKOFF: Duration = Duration::ZERO;
/// Candidates that hold no claim slot, so they claim in kind, day and band order.
fn idle(run_ids: &[RunId]) -> ClaimCandidates {
let candidates = run_ids
.iter()
.map(|&run_id| ClaimCandidate {
run_id,
load: SlotLoad::default(),
})
.collect();
ClaimCandidates::new(candidates).expect("every scenario claims from at least one run")
}
/// Discovery honors an `Only` allowlist plus the teams the access record has granted for 120 s, and
/// `All` admits every eligible run regardless of team, trigger, or already-seeding status.
#[tokio::test]
async fn discovery_scopes_to_the_allowlist_and_all_admits_every_run() -> Result<()> {
with_db(|pool| async move {
let self_run = insert_run(
&pool,
2,
"team_enablement",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
let other_team_run = insert_run(
&pool,
3,
"team_enablement",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
let dr_run = insert_run(
&pool,
4,
"disaster_recovery",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
let cross_team_run =
insert_run(&pool, 5, "team_enablement", "seeding", true, empty_pinned()).await?;
// Grant ages sit 30 s either side of the 120 s wait, so a changed wait fails this test.
for (team_id, level, granted_ago) in [
(6, "processing", Duration::from_secs(150)),
(7, "serving", Duration::from_secs(150)),
(8, "processing", Duration::from_secs(90)),
(9, "off", Duration::from_secs(150)),
] {
insert_access(&pool, team_id, level, granted_ago).await?;
insert_run(
&pool,
team_id,
"team_enablement",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
}
let only_team_two: TeamAllowlist = "2".parse().map_err(anyhow::Error::msg)?;
let discovered = discover_runs(&pool, &only_team_two, &[RunKind::Behavioral]).await?;
let mut discovered_teams = discovered
.iter()
.map(|run| run.team_id.0)
.collect::<Vec<_>>();
discovered_teams.sort_unstable();
ensure!(discovered_teams == [2, 6, 7], "{discovered_teams:?}");
let all_runs = discover_runs(&pool, &TeamAllowlist::All, &[RunKind::Behavioral]).await?;
ensure!(all_runs.iter().any(|run| run.run_id == self_run));
ensure!(all_runs.iter().any(|run| run.run_id == other_team_run));
ensure!(all_runs.iter().any(|run| run.run_id == cross_team_run));
ensure!(all_runs.iter().any(|run| run.run_id == dr_run));
Ok(())
})
.await
}
/// Two racing `establish_boundary` calls on the same awaiting-boundary run: the CAS admits exactly
/// one `Established`, and the loser re-reads the now-seeding row as `AlreadyEstablished`.
#[tokio::test]
async fn concurrent_boundary_establishment_promotes_exactly_one_run() -> Result<()> {
with_db(|pool| async move {
insert_run(
&pool,
2,
"team_enablement",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
let only_team_two: TeamAllowlist = "2".parse().map_err(anyhow::Error::msg)?;
let discovered = discover_runs(&pool, &only_team_two, &[RunKind::Behavioral]).await?;
let left = discovered[0].clone();
let right = discovered[0].clone();
let (left, right) = tokio::join!(
establish_boundary(&pool, left),
establish_boundary(&pool, right)
);
let outcomes = [left?, right?];
ensure!(
outcomes
.iter()
.filter(|outcome| matches!(outcome, BoundaryOutcome::Established(_)))
.count()
== 1
);
ensure!(
outcomes
.iter()
.filter(|outcome| matches!(outcome, BoundaryOutcome::AlreadyEstablished(_)))
.count()
== 1
);
Ok(())
})
.await
}
/// Warning notes append newest-first, dedupe on repeat, truncate the `error` column to the limit,
/// and are refused once a run leaves the `seeding` state.
#[tokio::test]
async fn run_warnings_dedupe_append_truncate_and_skip_terminal_runs() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
sqlx::query("UPDATE cohort_backfill_runs SET error = repeat('x', 5000) WHERE id = $1")
.bind(seeding_run)
.execute(&pool)
.await?;
ensure!(record_run_warning(&pool, seeding_run, RunWarningNote::ConditionsDropped).await?);
ensure!(!record_run_warning(&pool, seeding_run, RunWarningNote::ConditionsDropped).await?);
ensure!(record_run_warning(&pool, seeding_run, RunWarningNote::LookbackTruncated).await?);
let run_warnings: String =
sqlx::query_scalar("SELECT error FROM cohort_backfill_runs WHERE id = $1")
.bind(seeding_run)
.fetch_one(&pool)
.await?;
ensure!(run_warnings.len() == 4_096);
ensure!(
run_warnings
.matches(RunWarningNote::ConditionsDropped.as_str())
.count()
== 1
);
ensure!(
run_warnings
.matches(RunWarningNote::LookbackTruncated.as_str())
.count()
== 1
);
let terminal_run =
insert_run(&pool, 3, "team_enablement", "failed", false, empty_pinned()).await?;
ensure!(!record_run_warning(&pool, terminal_run, RunWarningNote::ConditionsDropped).await?);
Ok(())
})
.await
}
/// Loading the pinned payload drops superseded conditions (keeping only the live one plus a
/// `ConditionSuperseded` warning), rejects cohorts stored under another team, and refuses a
/// disaster-recovery run whose boundary was never pinned.
#[tokio::test]
async fn load_pinned_drops_superseded_and_rejects_cross_team_and_dr() -> Result<()> {
with_db(|pool| async move {
let pinned = json!({
"schema_version": 1,
"conditions": [
pinned_condition(10, ACTIVE_HASH, "active-event"),
pinned_condition(11, SUPERSEDED_HASH, "superseded-event"),
],
"event_names": ["active-event", "superseded-event"],
});
let self_run = insert_run(
&pool,
2,
"team_enablement",
"awaiting_boundary",
false,
pinned,
)
.await?;
insert_participation(
&pool,
self_run,
2,
10,
false,
behavioral_filter(ACTIVE_HASH, "active-event"),
)
.await?;
insert_participation(
&pool,
self_run,
2,
11,
true,
behavioral_filter(SUPERSEDED_HASH, "superseded-event"),
)
.await?;
let dr_run = insert_run(
&pool,
4,
"disaster_recovery",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
let cross_team_run =
insert_run(&pool, 5, "team_enablement", "seeding", true, empty_pinned()).await?;
insert_participation(
&pool,
cross_team_run,
999,
50,
true,
json!({"properties": {"type": "AND", "values": []}}),
)
.await?;
let all_runs = discover_runs(&pool, &TeamAllowlist::All, &[RunKind::Behavioral]).await?;
let self_discovered = all_runs
.iter()
.find(|run| run.run_id == self_run)
.cloned()
.context("self run was not discovered")?;
let cross_team_discovered = all_runs
.iter()
.find(|run| run.run_id == cross_team_run)
.cloned()
.context("cross-team run was not discovered")?;
let dr_discovered = all_runs
.into_iter()
.find(|run| run.run_id == dr_run)
.context("DR run was not discovered")?;
let seedable = match establish_boundary(&pool, self_discovered).await? {
BoundaryOutcome::Established(run) | BoundaryOutcome::AlreadyEstablished(run) => run,
BoundaryOutcome::NoLongerSeedable { .. } => bail!("self run was not seedable"),
};
let validated = seedable.load_pinned(&pool).await?;
ensure!(validated.run.conditions.len() == 1);
ensure!(validated.run.conditions[0].cohort_id == CohortId(10));
ensure!(
validated
.run
.event_names
.iter()
.map(String::as_str)
.collect::<Vec<_>>()
== ["active-event"]
);
ensure!(validated
.run
.filters
.behavioral_by_event_name
.contains_key("active-event"));
ensure!(!validated
.run
.filters
.behavioral_by_event_name
.contains_key("superseded-event"));
ensure!(validated.warnings.iter().any(|warning| matches!(
warning,
PinnedWarning::ConditionSuperseded { cohort_id, .. } if *cohort_id == CohortId(11)
)));
// Already seeding with a boundary, so it promotes to a SeedableRun without a DB write.
let cross_team_seedable = match establish_boundary(&pool, cross_team_discovered).await? {
BoundaryOutcome::Established(run) | BoundaryOutcome::AlreadyEstablished(run) => run,
BoundaryOutcome::NoLongerSeedable { .. } => bail!("cross-team run was not seedable"),
};
ensure!(matches!(
cross_team_seedable.load_pinned(&pool).await,
Err(RunError::CrossTeamParticipation {
actual_team_id: 999,
..
})
));
ensure!(matches!(
establish_boundary(&pool, dr_discovered).await,
Err(RunError::DisasterRecoveryBoundaryMissing(id)) if id == dr_run
));
Ok(())
})
.await
}
/// Manual dispatch reads only active participations, carries the behavioral-only shape hash onto
/// the control tile, and refuses stale run states or malformed tenant/hash data before Kafka.
#[tokio::test]
async fn reconcile_run_load_is_fail_closed_and_behavioral_hash_scoped() -> Result<()> {
with_db(|pool| async move {
let run_id = insert_run(
&pool,
20,
"team_enablement",
"seeding",
true,
empty_pinned(),
)
.await?;
insert_participation(
&pool,
run_id,
20,
201,
false,
behavioral_filter(ACTIVE_HASH, "active-event"),
)
.await?;
insert_participation(
&pool,
run_id,
20,
202,
true,
behavioral_filter(SUPERSEDED_HASH, "superseded-event"),
)
.await?;
let register_backfill = RegisterBackfillConfirmation::confirmed_by_operator();
// A seeding run that never planned has no completion proof, so a complete dispatch fails
// closed (the CLI must use --allow-incomplete for an unplanned run).
ensure!(matches!(
prepare_reconcile_dispatch(
&pool,
run_id,
CompletionRequirement::Complete,
register_backfill,
)
.await,
Err(PrepareReconcileDispatchError::PlanningUnproven(id)) if id == run_id
));
let overridden = prepare_reconcile_dispatch(
&pool,
run_id,
CompletionRequirement::AllowIncomplete,
register_backfill,
)
.await?;
ensure!(overridden.total_chunks() == 0);
ensure!(overridden.remaining_chunks() == 0);
let reconcile = load_reconcile_run(&pool, run_id).await?;
ensure!(reconcile.run_id() == run_id);
ensure!(reconcile.status() == RunStatus::Seeding);
ensure!(reconcile.cohort_count() == 1);
let tile = reconcile
.tiles()
.next()
.context("missing active cohort tile")?;
ensure!(tile.team_id() == TeamId(20));
ensure!(tile.cohort_id() == CohortId(201));
ensure!(tile.scope().hash_str() == "behavioral-shape");
ensure!(tile.scope().kind() == ScopeKind::Behavioral);
ensure!(tile.run_id() == run_id);
sqlx::query("UPDATE cohort_backfill_runs SET status = 'reconciling' WHERE id = $1")
.bind(run_id)
.execute(&pool)
.await?;
ensure!(load_reconcile_run(&pool, run_id).await?.status() == RunStatus::Reconciling);
// Reaching `reconciling` is not itself a planning proof — only the stamp is.
ensure!(matches!(
prepare_reconcile_dispatch(
&pool,
run_id,
CompletionRequirement::Complete,
register_backfill,
)
.await,
Err(PrepareReconcileDispatchError::PlanningUnproven(id)) if id == run_id
));
sqlx::query("UPDATE cohort_backfill_runs SET chunks_planned_at = now() WHERE id = $1")
.bind(run_id)
.execute(&pool)
.await?;
ensure!(prepare_reconcile_dispatch(
&pool,
run_id,
CompletionRequirement::Complete,
register_backfill,
)
.await
.is_ok());
sqlx::query(
"UPDATE cohort_backfill_run_cohorts SET behavioral_filters_shape_hash = '' \
WHERE run_id = $1 AND cohort_id = 201",
)
.bind(run_id)
.execute(&pool)
.await?;
ensure!(matches!(
load_reconcile_run(&pool, run_id).await,
Err(ReconcileRunError::InvalidShapeHash { cohort_id, kind, .. })
if cohort_id == CohortId(201) && kind == ScopeKind::Behavioral
));
sqlx::query(
"UPDATE cohort_backfill_run_cohorts \
SET behavioral_filters_shape_hash = 'behavioral-shape', team_id = 999 \
WHERE run_id = $1 AND cohort_id = 201",
)
.bind(run_id)
.execute(&pool)
.await?;
ensure!(matches!(
load_reconcile_run(&pool, run_id).await,
Err(ReconcileRunError::CrossTeamParticipation {
expected_team_id: 20,
actual_team_id: 999,
..
})
));
sqlx::query(
"UPDATE cohort_backfill_run_cohorts SET superseded_at = now() WHERE run_id = $1",
)
.bind(run_id)
.execute(&pool)
.await?;
ensure!(matches!(
load_reconcile_run(&pool, run_id).await,
Err(ReconcileRunError::NoActiveParticipations(id)) if id == run_id
));
let completed_run = insert_run(
&pool,
21,
"team_enablement",
"completed",
true,
empty_pinned(),
)
.await?;
insert_participation(
&pool,
completed_run,
21,
211,
false,
behavioral_filter(ACTIVE_HASH, "active-event"),
)
.await?;
ensure!(matches!(
load_reconcile_run(&pool, completed_run).await,
Err(ReconcileRunError::Status {
status: RunStatus::Completed,
..
})
));
Ok(())
})
.await
}
/// Planning a seeding run's days is idempotent (re-planning inserts nothing), stamps the run's team
/// onto every chunk, and reports `RunNotSeeding` for a run that is not seeding.
#[tokio::test]
async fn planning_is_idempotent_scopes_team_and_gates_on_seeding() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND)
.await?
)? == 2
);
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND)
.await?
)? == 0
);
let progress = store.chunk_progress(seeding_run).await?;
ensure!(progress.total() == 2);
ensure!(progress.remaining() == 2);
let chunk_teams_match: bool = sqlx::query_scalar(
"SELECT bool_and(team_id = 2) FROM cohort_backfill_chunks WHERE run_id = $1",
)
.bind(seeding_run)
.fetch_one(&pool)
.await?;
ensure!(chunk_teams_match);
let idle_run = insert_run(
&pool,
3,
"team_enablement",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
ensure!(matches!(
store
.plan_chunks(idle_run, historical([200]), ONE_BAND)
.await?,
PlanOutcome::RunNotSeeding
));
Ok(())
})
.await
}
/// A multi-band plan fans a day into one chunk per band (idempotently), and a claim decodes the
/// band with the day's full band count so the scan predicate partitions persons correctly.
#[tokio::test]
async fn planning_fans_out_bands_and_claims_carry_the_band_count() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let four_bands = NonZeroU16::new(4).context("four is non-zero")?;
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100]), four_bands)
.await?
)? == 4
);
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100]), four_bands)
.await?
)? == 0
);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let claimed = store
.claim_next(
&idle(&[seeding_run]),
&Claimant::new("worker-a")?,
lease60,
attempts5,
)
.await?
.context("claimant found no chunk")?;
let band = claimed.chunk.spec().band;
ensure!(band.num_bands().get() == 4);
ensure!(band.band() < 4);
drop(claimed);
Ok(())
})
.await
}
/// Two racing claimants take disjoint chunks — `SKIP LOCKED` never hands the same chunk to both.
#[tokio::test]
async fn concurrent_claims_take_disjoint_chunks() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND)
.await?
)? == 2
);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let candidates = idle(&[seeding_run]);
let first_store = store.clone();
let second_store = store.clone();
let claimant_a = Claimant::new("worker-a")?;
let claimant_b = Claimant::new("worker-b")?;
let (first, second) = tokio::join!(
first_store.claim_next(&candidates, &claimant_a, lease60, attempts5),
second_store.claim_next(&candidates, &claimant_b, lease60, attempts5),
);
let first = first?.context("first claimant found no chunk")?;
let second = second?.context("second claimant found no chunk")?;
ensure!(first.chunk.spec().lease.chunk_id() != second.chunk.spec().lease.chunk_id());
Ok(())
})
.await
}
/// An expired lease is reclaimed as the SAME chunk with a bumped epoch; the reclaim fences the stale
/// lease so every one of its fenced operations reports `LeaseLost`, while the fresh lease works.
#[tokio::test]
async fn expired_lease_reclaim_bumps_epoch_and_fences_the_stale_lease() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let first = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
.await?
.context("first claimant found no chunk")?;
let stale = first.chunk.spec().lease;
drop(first);
sqlx::query(
"UPDATE cohort_backfill_chunks SET lease_expires_at = now() - interval '1 second' WHERE id = $1",
)
.bind(stale.chunk_id())
.execute(&pool)
.await?;
let reclaimed = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
.await?
.context("expired chunk was not reclaimed")?;
let reclaimed_lease = reclaimed.chunk.spec().lease;
ensure!(reclaimed_lease.chunk_id() == stale.chunk_id());
ensure!(reclaimed_lease.epoch() == ClaimEpoch(stale.epoch().0 + 1));
ensure_lease_lost(
test_support::heartbeat(&store, stale, &Claimant::new("worker-a")?, lease60).await,
)?;
ensure_lease_lost(test_support::mark_produced_raw(&store, stale, 1, ScanVolume::default()).await)?;
ensure_lease_lost(test_support::confirm_raw(&store, stale, &ProduceHwms::default()).await)?;
ensure_lease_lost(test_support::fail(&store, stale, "stale failure", NO_BACKOFF).await)?;
ensure_lease_lost(test_support::unclaim(&store, stale).await)?;
// The fresh lease, in contrast, drives the chunk through mark-produced and confirm.
let mut hwms = ProduceHwms::default();
hwms.observe(3, 41);
test_support::mark_produced_raw(&store, reclaimed_lease, 0, ScanVolume::default()).await?;
test_support::confirm_raw(&store, reclaimed_lease, &hwms).await?;
drop(reclaimed);
Ok(())
})
.await
}
/// Unclaiming a scanning chunk returns it to `pending` and refunds the attempt the claim charged.
#[tokio::test]
async fn unclaim_returns_chunk_to_pending_and_refunds_one_attempt() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100]), ONE_BAND)
.await?
)? == 1
);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let claimed = store
.claim_next(
&idle(&run_ids),
&Claimant::new("worker-a")?,
lease60,
attempts5,
)
.await?
.context("claimant found no chunk")?;
let lease = claimed.chunk.spec().lease;
let chunk_id = lease.chunk_id();
sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 5 WHERE id = $1")
.bind(chunk_id)
.execute(&pool)
.await?;
store.unclaim(lease).await?;
drop(claimed);
let (pending_status, refunded_attempts): (String, i32) =
sqlx::query_as("SELECT status, attempts FROM cohort_backfill_chunks WHERE id = $1")
.bind(chunk_id)
.fetch_one(&pool)
.await?;
ensure!(pending_status == "pending");
ensure!(refunded_attempts == 4);
Ok(())
})
.await
}
/// A failed chunk is held out of the claim gate until the wait `fail` stamped has elapsed, then
/// becomes claimable again with the stamp cleared. Without the hold, the poll loop re-claims a
/// chunk failing for a durable reason within seconds and burns its whole attempt budget on the same
/// failure, which is the reclaim storm this gate exists to break.
#[tokio::test]
async fn a_failed_chunk_waits_out_its_backoff_before_it_is_claimable_again() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let claimed = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
.await?
.context("claimant found no chunk")?;
let lease = claimed.chunk.spec().lease;
let chunk_id = lease.chunk_id();
// The claim reports the attempt the backoff is sized from.
ensure!(claimed.chunk.spec().attempt.get() == 1);
store
.fail(
lease,
&RenderedError::from_message("transient"),
Duration::from_secs(600),
)
.await?;
drop(claimed);
// The stamp is `now() + delay`, so the remaining wait is the delay minus test runtime.
let remaining_secs: f64 = sqlx::query_scalar(
"SELECT extract(epoch FROM next_attempt_at - now())::float8
FROM cohort_backfill_chunks WHERE id = $1",
)
.bind(chunk_id)
.fetch_one(&pool)
.await?;
ensure!(
(540.0..=600.0).contains(&remaining_secs),
"expected a ~600s hold, got {remaining_secs}s"
);
ensure!(store
.claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
.await?
.is_none());
// Once the wait has passed the chunk is claimable, and the claim clears the stamp so a
// later failure is not gated by a stale one.
sqlx::query(
"UPDATE cohort_backfill_chunks SET next_attempt_at = now() - interval '1 second' WHERE id = $1",
)
.bind(chunk_id)
.execute(&pool)
.await?;
let retried = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
.await?
.context("chunk past its backoff was not claimable")?;
ensure!(retried.chunk.spec().attempt.get() == 2);
let stamp: Option<chrono::DateTime<chrono::Utc>> =
sqlx::query_scalar("SELECT next_attempt_at FROM cohort_backfill_chunks WHERE id = $1")
.bind(chunk_id)
.fetch_one(&pool)
.await?;
ensure!(stamp.is_none(), "the claim left a stale backoff stamp");
Ok(())
})
.await
}
/// The recovery path sizes the wait from the chunk's own attempt count, and that wait is what
/// reaches `next_attempt_at`.
///
/// The store's `fail` takes the delay as an argument, so every other test here supplies its own and
/// proves nothing about where a real one comes from. A recovery path that always asked for the
/// first attempt's wait would hold every chunk a flat `base` forever — defeating the whole feature
/// — while the policy's own unit tests and the gate test above all stayed green.
///
/// The chunks start one attempt below the cap, so the claim reports an attempt whose ceiling has
/// saturated at 1800s while a first attempt's is 1s. Full jitter draws from `[0, ceiling]`, so one
/// sample could still land low; over four chunks the chance that all four fall inside the first
/// attempt's 1s bound is about (1/1800)^4, which does not flake.
#[tokio::test]
async fn the_recovery_path_draws_its_wait_from_the_chunks_attempt_count() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts50 = MaxAttempts::new(50)?;
let run_ids = [seeding_run];
let policy =
RetryBackoffPolicy::new(Duration::from_secs(1), Duration::from_secs(1800)).unwrap();
let days = [100, 101, 102, 103];
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical(days), ONE_BAND)
.await?
)? == 4
);
sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 48 WHERE run_id = $1")
.bind(seeding_run)
.execute(&pool)
.await?;
let mut longest_wait = Duration::ZERO;
for day in days {
let claimed = store
.claim_next(
&idle(&run_ids),
&Claimant::new("worker-a")?,
lease60,
attempts50,
)
.await?
.context("a chunk under the cap was not claimable")?;
let chunk = claimed.chunk;
let lease = claimed.lease;
let chunk_id = chunk.spec().lease.chunk_id();
ensure!(chunk.spec().attempt.get() == 49, "day {day}");
let drawn =
test_support::fail_via_recovery(&store, chunk, "clickhouse memory limit", policy)
.await
.context("the recovery path did not fail the chunk")?;
drop(lease);
// The stamp is `now() + delay`, so the remaining wait is the delay minus test runtime.
let remaining_secs: f64 = sqlx::query_scalar(
"SELECT extract(epoch FROM next_attempt_at - now())::float8
FROM cohort_backfill_chunks WHERE id = $1",
)
.bind(chunk_id)
.fetch_one(&pool)
.await?;
ensure!(
remaining_secs <= drawn.as_secs_f64(),
"day {day} stamped {remaining_secs}s, more than the {drawn:?} the path drew"
);
longest_wait = longest_wait.max(drawn);
}
ensure!(
longest_wait > policy.ceiling(AttemptCount::from_row(1)),
"the longest of four near-cap waits was {longest_wait:?}, inside the first attempt's \
ceiling — the recovery path is not reading the attempt count"
);
Ok(())
})
.await
}
/// Only the failed→claim transition reads the backoff stamp. A `pending` chunk has not failed, and
/// an expired `produced` lease must reclaim immediately because its tiles are already in Kafka and
/// the run cannot complete until it reaches `confirmed`. Gating either would stall a run on a
/// column that describes neither.
///
/// The two lease legs are a guard rail, not a live bug. The claim `UPDATE` sets `next_attempt_at`
/// to NULL in the same statement that writes `scanning`, so no reclaimable row carries a stale
/// stamp today and the arms would pass with or without their gate. Each leg stamps one by hand, so
/// a later edit that widens the gate onto a lease arm is caught here instead of stalling a run in
/// production.
#[tokio::test]
async fn the_backoff_gate_applies_only_to_the_failed_claim_arm() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
// A pending chunk carrying a future stamp is still claimable.
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
sqlx::query(
"UPDATE cohort_backfill_chunks SET next_attempt_at = now() + interval '1 hour' WHERE run_id = $1",
)
.bind(seeding_run)
.execute(&pool)
.await?;
let pending = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
.await?
.context("a pending chunk was gated by the backoff stamp")?;
let pending_lease = pending.chunk.spec().lease;
// An unclaim returns it to `pending`, and it stays claimable however the stamp reads.
store.unclaim(pending_lease).await?;
drop(pending);
sqlx::query(
"UPDATE cohort_backfill_chunks SET next_attempt_at = now() + interval '1 hour' WHERE id = $1",
)
.bind(pending_lease.chunk_id())
.execute(&pool)
.await?;
let reclaimed = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
.await?
.context("an unclaimed chunk was gated by the backoff stamp")?;
// A produced chunk whose lease expired reclaims regardless of the stamp.
let produced_lease = reclaimed.chunk.spec().lease;
test_support::mark_produced_raw(&store, produced_lease, 1, ScanVolume::default()).await?;
drop(reclaimed);
sqlx::query(
"UPDATE cohort_backfill_chunks
SET lease_expires_at = now() - interval '1 second',
next_attempt_at = now() + interval '1 hour'
WHERE id = $1",
)
.bind(produced_lease.chunk_id())
.execute(&pool)
.await?;
let produced_reclaim = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
.await?
.context("an expired produced lease was gated by the backoff stamp")?;
// A scanning chunk whose lease expired reclaims too. It is left `scanning` rather than
// marked produced, which is the shape of a worker that died before it reached `fail`, so
// nothing sized a wait for it and nothing should hold it back.
let scanning_lease = produced_reclaim.chunk.spec().lease;
drop(produced_reclaim);
sqlx::query(
"UPDATE cohort_backfill_chunks
SET lease_expires_at = now() - interval '1 second',
next_attempt_at = now() + interval '1 hour'
WHERE id = $1",
)
.bind(scanning_lease.chunk_id())
.execute(&pool)
.await?;
ensure!(store
.claim_next(&idle(&run_ids), &Claimant::new("worker-d")?, lease60, attempts5)
.await?
.is_some());
Ok(())
})
.await
}
/// A trailing day is planned with its hold and claimed only once the hold lapses and its run is
/// `trailing`. Discovery admits the trailing run once a hold lapses, prepare keeps it eligible, and
/// the run completes once that day confirms.
#[tokio::test]
async fn a_trailing_day_waits_out_its_hold_and_completes_its_trailing_run() -> Result<()> {
with_db(|pool| async move {
let run_id =
insert_run(&pool, 2, "cohort_created", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let claimant = Claimant::new("worker-a")?;
let run_ids = [run_id];
let behavioral = [RunKind::Behavioral];
let hold = Utc::now() + chrono::Duration::hours(1);
let [historical_day] = historical([100]);
ensure!(
planned_count(
store
.plan_chunks(run_id, [historical_day, trailing(101, hold)], ONE_BAND)
.await?
)? == 2
);
let holds: Vec<Option<DateTime<Utc>>> = sqlx::query_scalar(
"SELECT claimable_after FROM cohort_backfill_chunks WHERE run_id = $1 ORDER BY day",
)
.bind(run_id)
.fetch_all(&pool)
.await?;
ensure!(
holds
== [
None,
DateTime::from_timestamp_millis(hold.timestamp_millis())
],
"unexpected holds {holds:?}"
);
let historical_claim = store
.claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
.await?
.context("the historical day was not claimable")?;
ensure!(historical_claim.chunk.spec().day == 100);
ensure!(
store
.claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
.await?
.is_none(),
"the trailing day was claimed before its hold lapsed"
);
let historical_lease = historical_claim.chunk.spec().lease;
test_support::mark_produced_raw(&store, historical_lease, 0, ScanVolume::default()).await?;
test_support::confirm_raw(&store, historical_lease, &ProduceHwms::default()).await?;
drop(historical_claim);
// The finalizer's move once it has stamped readiness. Until a hold lapses nothing of the
// run can be claimed, so discovery leaves it alone.
sqlx::query("UPDATE cohort_backfill_runs SET status = 'trailing' WHERE id = $1")
.bind(run_id)
.execute(&pool)
.await?;
ensure!(
discover_runs(&pool, &TeamAllowlist::All, &behavioral)
.await?
.iter()
.all(|run| run.run_id != run_id),
"the trailing run was discovered before any of its holds lapsed"
);
sqlx::query(
"UPDATE cohort_backfill_chunks SET claimable_after = now() - interval '1 second'
WHERE run_id = $1 AND claimable_after IS NOT NULL",
)
.bind(run_id)
.execute(&pool)
.await?;
let resumed = discover_runs(&pool, &TeamAllowlist::All, &behavioral)
.await?
.into_iter()
.find(|run| run.run_id == run_id)
.context("discovery skipped the trailing run once its hold lapsed")?;
match establish_boundary(&pool, resumed).await? {
BoundaryOutcome::AlreadyEstablished(run) => ensure!(run.phase == SeedPhase::Trailing),
other => bail!("the trailing run resumed as {other:?}"),
}
ensure!(
eligible_run_ids(&pool, &store, &TeamAllowlist::All, &behavioral)
.await
.contains(&run_id),
"prepare did not keep the trailing run claim-eligible"
);
ensure!(complete_trailing_runs(&pool, &run_ids).await? == 0);
let trailing_claim = store
.claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
.await?
.context("the trailing day was not claimable after its hold lapsed")?;
ensure!(trailing_claim.chunk.spec().day == 101);
let trailing_lease = trailing_claim.chunk.spec().lease;
test_support::heartbeat(&store, trailing_lease, &claimant, lease60).await?;
test_support::mark_produced_raw(&store, trailing_lease, 0, ScanVolume::default()).await?;
test_support::confirm_raw(&store, trailing_lease, &ProduceHwms::default()).await?;
drop(trailing_claim);
ensure!(complete_trailing_runs(&pool, &run_ids).await? == 1);
let (status, finished): (String, bool) = sqlx::query_as(
"SELECT status, finished_at IS NOT NULL FROM cohort_backfill_runs WHERE id = $1",
)
.bind(run_id)
.fetch_one(&pool)
.await?;
ensure!(status == "completed" && finished);
Ok(())
})
.await
}
/// While a run still seeds, a held chunk stays unclaimable even after its hold lapses, so the
/// reconciling CAS cannot cut its lease mid-scan; and once every chunk has confirmed, the trailing
/// sweep leaves the run in `seeding`, since completing it there would skip `reconciling` and the
/// readiness stamp.
#[tokio::test]
async fn a_seeding_run_neither_hands_out_a_lapsed_held_chunk_nor_completes_as_trailing(
) -> Result<()> {
with_db(|pool| async move {
let run_id =
insert_run(&pool, 2, "cohort_created", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [run_id];
let [historical_day] = historical([100]);
let lapsed = Utc::now() - chrono::Duration::hours(1);
store
.plan_chunks(run_id, [historical_day, trailing(101, lapsed)], ONE_BAND)
.await?;
sqlx::query(
"UPDATE cohort_backfill_chunks SET status = 'confirmed'
WHERE run_id = $1 AND claimable_after IS NULL",
)
.bind(run_id)
.execute(&pool)
.await?;
ensure!(
store
.claim_next(
&idle(&run_ids),
&Claimant::new("worker-a")?,
lease60,
attempts5
)
.await?
.is_none(),
"a held chunk was claimed while its run was still seeding"
);
sqlx::query("UPDATE cohort_backfill_chunks SET status = 'confirmed' WHERE run_id = $1")
.bind(run_id)
.execute(&pool)
.await?;
ensure!(complete_trailing_runs(&pool, &run_ids).await? == 0);
let status: String =
sqlx::query_scalar("SELECT status FROM cohort_backfill_runs WHERE id = $1")
.bind(run_id)
.fetch_one(&pool)
.await?;
ensure!(
status == "seeding",
"the seeding run was completed as if trailing"
);
Ok(())
})
.await
}
/// The `scanning`→`produced` CAS records what the scan moved, and planning leaves the byte columns
/// at the database-side default. The default is load-bearing: `plan_chunks` inserts an explicit
/// column list that names neither column, so a Django-only default would make every planning insert
/// violate NOT NULL during a rollout.
#[tokio::test]
async fn mark_produced_records_the_scan_byte_volume_that_planning_defaults_to_zero() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
let planned: (i64, i64) = sqlx::query_as(
"SELECT scan_received_bytes, scan_decoded_bytes FROM cohort_backfill_chunks WHERE run_id = $1",
)
.bind(seeding_run)
.fetch_one(&pool)
.await?;
ensure!(planned == (0, 0));
let claimed = store
.claim_next(
&idle(&[seeding_run]),
&Claimant::new("worker-a")?,
LeaseDuration::new(Duration::from_secs(60))?,
MaxAttempts::new(5)?,
)
.await?
.context("claimant found no chunk")?;
let lease = claimed.chunk.spec().lease;
test_support::mark_produced_raw(
&store,
lease,
7,
ScanVolume::new(9_876_543_210, 41_234_567_890),
)
.await?;
drop(claimed);
let stored: (i64, i64, i64) = sqlx::query_as(
"SELECT tiles_produced, scan_received_bytes, scan_decoded_bytes
FROM cohort_backfill_chunks WHERE id = $1",
)
.bind(lease.chunk_id())
.fetch_one(&pool)
.await?;
ensure!(stored == (7, 9_876_543_210, 41_234_567_890));
Ok(())
})
.await
}
/// The attempt cap is terminal for a `failed` chunk (no further claim), but an expired `produced`
/// chunk sitting AT the cap is still reclaimed with a bumped epoch — its tiles are already in
/// Kafka, so it must keep retrying until it reaches `confirmed`. Only `scanning` reclaims are
/// capped (see `expired_scanning_chunk_at_the_cap_is_reaped_not_reclaimed`).
#[tokio::test]
async fn attempt_cap_is_terminal_for_failed_but_reclaims_expired_produced() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
// A chunk driven to the attempt cap by claiming, then failed, is not claimable again.
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 4 WHERE run_id = $1")
.bind(seeding_run)
.execute(&pool)
.await?;
let retry = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-d")?, lease60, attempts5)
.await?
.context("chunk under the cap was not claimable")?;
let retry_lease = retry.chunk.spec().lease;
let reclaimed_attempts: i32 =
sqlx::query_scalar("SELECT attempts FROM cohort_backfill_chunks WHERE id = $1")
.bind(retry_lease.chunk_id())
.fetch_one(&pool)
.await?;
ensure!(reclaimed_attempts == 5);
store
.fail(retry_lease, &RenderedError::from_message("terminal"), NO_BACKOFF)
.await?;
drop(retry);
ensure!(store
.claim_next(&idle(&run_ids), &Claimant::new("worker-terminal")?, lease60, attempts5)
.await?
.is_none());
// A second chunk, marked produced then expired at the cap, IS reclaimed.
ensure!(planned_count(store.plan_chunks(seeding_run, historical([101]), ONE_BAND).await?)? == 1);
let final_attempt = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-e")?, lease60, attempts5)
.await?
.context("second chunk was not claimable")?;
let final_attempt_lease = final_attempt.chunk.spec().lease;
test_support::mark_produced_raw(&store, final_attempt_lease, 1, ScanVolume::default()).await?;
drop(final_attempt);
sqlx::query(
"UPDATE cohort_backfill_chunks SET attempts = 5, lease_expires_at = now() - interval '1 second' WHERE id = $1",
)
.bind(final_attempt_lease.chunk_id())
.execute(&pool)
.await?;
let observed = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-f")?, lease60, attempts5)
.await?
.context("expired produced chunk at the attempt cap was not reclaimed")?;
let observed_lease = observed.chunk.spec().lease;
ensure!(observed_lease.chunk_id() == final_attempt_lease.chunk_id());
ensure!(observed_lease.epoch() == ClaimEpoch(final_attempt_lease.epoch().0 + 1));
let active_reclaim_attempts: i32 =
sqlx::query_scalar("SELECT attempts FROM cohort_backfill_chunks WHERE id = $1")
.bind(observed_lease.chunk_id())
.fetch_one(&pool)
.await?;
ensure!(active_reclaim_attempts == 5);
// The claim `CASE` stops incrementing at the cap, so this is the arm where the count the
// spec reports and the number of claims diverge. The spec must still carry the column, not
// the claim tally: the retry backoff is sized from it.
ensure!(observed.chunk.spec().attempt.get() == 5);
Ok(())
})
.await
}
/// A `scanning` chunk whose lease expires below the attempt cap is left for reclaim, but once its
/// attempts saturate it is neither reclaimed (no hard-crash loop) nor left invisible: the reaper
/// dead-letters it to `failed` — the same terminal state a clean cap-out reaches — exactly once.
#[tokio::test]
async fn expired_scanning_chunk_at_the_cap_is_reaped_not_reclaimed() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let claimed = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-a")?, lease60, attempts5)
.await?
.context("claimant found no chunk")?;
let chunk_id = claimed.chunk.spec().lease.chunk_id();
drop(claimed);
// Expired below the cap: not reaped — the ordinary reclaim path still owns it.
sqlx::query(
"UPDATE cohort_backfill_chunks SET lease_expires_at = now() - interval '1 second' WHERE id = $1",
)
.bind(chunk_id)
.execute(&pool)
.await?;
ensure!(store.reap_poisoned_chunks(&run_ids, attempts5).await? == 0);
// A hard-crashed worker's residue: still `scanning`, attempts saturated, lease expired.
sqlx::query("UPDATE cohort_backfill_chunks SET attempts = 5 WHERE id = $1")
.bind(chunk_id)
.execute(&pool)
.await?;
ensure!(store
.claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
.await?
.is_none());
ensure!(store.reap_poisoned_chunks(&run_ids, attempts5).await? == 1);
let (status, last_error): (String, String) =
sqlx::query_as("SELECT status, last_error FROM cohort_backfill_chunks WHERE id = $1")
.bind(chunk_id)
.fetch_one(&pool)
.await?;
ensure!(status == "failed");
ensure!(last_error.contains("attempt cap"));
// Terminal: not claimable, and a second sweep is a no-op.
ensure!(store
.claim_next(&idle(&run_ids), &Claimant::new("worker-c")?, lease60, attempts5)
.await?
.is_none());
ensure!(store.reap_poisoned_chunks(&run_ids, attempts5).await? == 0);
Ok(())
})
.await
}
/// Only chunks that saturated the attempt cap count as exhausted. A `failed` chunk still under the
/// cap is reclaimable and will retry, so reporting it would fail runs that were about to recover.
#[tokio::test]
async fn runs_with_exhausted_chunks_selects_only_capped_failures() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND).await?)? == 2);
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let chunk_ids: Vec<Uuid> = sqlx::query_scalar(
"SELECT id FROM cohort_backfill_chunks WHERE run_id = $1 ORDER BY day",
)
.bind(seeding_run)
.fetch_all(&pool)
.await?;
sqlx::query(
"UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 4, last_error = 'still retrying' WHERE id = $1",
)
.bind(chunk_ids[0])
.execute(&pool)
.await?;
ensure!(store
.runs_with_exhausted_chunks(&run_ids, attempts5)
.await?
.is_empty());
sqlx::query(
"UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5, last_error = 'scan blew up' WHERE id = $1",
)
.bind(chunk_ids[1])
.execute(&pool)
.await?;
let exhausted = store.runs_with_exhausted_chunks(&run_ids, attempts5).await?;
ensure!(exhausted.len() == 1);
ensure!(exhausted[0].run_id == seeding_run);
ensure!(exhausted[0].exhausted == 1, "only the capped chunk counts");
ensure!(exhausted[0].chunk_id == chunk_ids[1].to_string());
ensure!(exhausted[0].last_error == "scan blew up");
ensure!(store
.runs_with_exhausted_chunks(&[], attempts5)
.await?
.is_empty());
Ok(())
})
.await
}
/// The reported chunk and error come from one row. Aggregating them independently pairs the lowest
/// chunk id with some other chunk's error, sending the operator to read a row that never failed
/// that way.
#[tokio::test]
async fn exhausted_chunk_and_error_are_read_off_the_same_row() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100, 101]), ONE_BAND).await?)? == 2);
let attempts5 = MaxAttempts::new(5)?;
// Ordered so the lowest-id chunk carries the *higher* error text: an independent
// `min(last_error)` would then hand back the other chunk's error.
let mut chunk_ids: Vec<Uuid> =
sqlx::query_scalar("SELECT id FROM cohort_backfill_chunks WHERE run_id = $1")
.bind(seeding_run)
.fetch_all(&pool)
.await?;
chunk_ids.sort();
for (chunk_id, error) in chunk_ids.iter().zip(["zzz lowest id", "aaa highest id"]) {
sqlx::query(
"UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5, last_error = $2 WHERE id = $1",
)
.bind(chunk_id)
.bind(error)
.execute(&pool)
.await?;
}
let exhausted = store
.runs_with_exhausted_chunks(&[seeding_run], attempts5)
.await?;
ensure!(exhausted.len() == 1);
ensure!(exhausted[0].exhausted == 2, "both capped chunks counted");
ensure!(exhausted[0].chunk_id == chunk_ids[0].to_string());
ensure!(exhausted[0].last_error == "zzz lowest id");
// A capped chunk with no persisted error still has to render as something: the run error
// interpolates this text, and an empty one leaves the operator a trailing colon.
sqlx::query("UPDATE cohort_backfill_chunks SET last_error = '' WHERE id = $1")
.bind(chunk_ids[0])
.execute(&pool)
.await?;
let exhausted = store
.runs_with_exhausted_chunks(&[seeding_run], attempts5)
.await?;
ensure!(exhausted[0].last_error == NO_ERROR_RECORDED);
Ok(())
})
.await
}
/// Readiness does not wait for a trailing day, so that day's exhausted chunk fails its run only once
/// the run has stamped readiness and is `trailing`.
#[tokio::test]
async fn an_exhausted_trailing_chunk_fails_its_run_only_once_the_run_is_trailing() -> Result<()> {
with_db(|pool| async move {
let run_id =
insert_run(&pool, 2, "cohort_created", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [run_id];
store
.plan_chunks(run_id, [trailing(101, Utc::now())], ONE_BAND)
.await?;
sqlx::query(
"UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5 WHERE run_id = $1",
)
.bind(run_id)
.execute(&pool)
.await?;
ensure!(
fail_exhausted_runs_of_kind(&pool, &store, &run_ids, RunKind::Behavioral, attempts5)
.await
== 0
);
sqlx::query("UPDATE cohort_backfill_runs SET status = 'trailing' WHERE id = $1")
.bind(run_id)
.execute(&pool)
.await?;
ensure!(
fail_exhausted_runs_of_kind(&pool, &store, &run_ids, RunKind::Behavioral, attempts5)
.await
== 1
);
Ok(())
})
.await
}
/// Failing a run whose chunk exhausted its retries stops the run dead: a still-pending sibling is no
/// longer claimable, a live sibling's heartbeat is refused, and a second failure is a no-op rather
/// than a double-count. Without it the run sits in `seeding` forever holding its cohort's slot.
#[tokio::test]
async fn exhausted_chunk_fails_the_run_and_stops_further_claims() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(
planned_count(store.plan_chunks(seeding_run, historical([100, 101, 102]), ONE_BAND).await?)? == 3
);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let claimant = Claimant::new("worker-a")?;
// One chunk mid-scan with a live lease, one capped out, one left pending.
let claimed = store
.claim_next(&idle(&run_ids), &claimant, lease60, attempts5)
.await?
.context("claimant found no chunk")?;
let live_lease = claimed.chunk.spec().lease;
let capped_id: Uuid = sqlx::query_scalar(
"SELECT id FROM cohort_backfill_chunks WHERE run_id = $1 AND status = 'pending' ORDER BY day LIMIT 1",
)
.bind(seeding_run)
.fetch_one(&pool)
.await?;
sqlx::query(
"UPDATE cohort_backfill_chunks SET status = 'failed', attempts = 5, last_error = 'scan blew up' WHERE id = $1",
)
.bind(capped_id)
.execute(&pool)
.await?;
// Through the seeder's own pass, not a hand-rolled equivalent: the scan, the error text and
// the attempt-cap interpolation are what an operator ends up reading.
ensure!(
fail_exhausted_runs_of_kind(
&pool,
&store,
&run_ids,
RunKind::Behavioral,
attempts5
)
.await
== 1
);
let (status, error, finished): (String, String, bool) = sqlx::query_as(
"SELECT status, error, finished_at IS NOT NULL FROM cohort_backfill_runs WHERE id = $1",
)
.bind(seeding_run)
.fetch_one(&pool)
.await?;
ensure!(status == "failed");
ensure!(error.contains("1 chunk(s) exhausted the 5 attempt retry budget"));
ensure!(error.contains(&capped_id.to_string()));
ensure!(error.contains("scan blew up"));
ensure!(finished);
// The claim and heartbeat predicates both join `runs.status = 'seeding'`, so the pending
// sibling stops being claimable and the in-flight one halts on its next beat.
ensure!(store
.claim_next(&idle(&run_ids), &Claimant::new("worker-b")?, lease60, attempts5)
.await?
.is_none());
let still_pending: i64 = sqlx::query_scalar(
"SELECT count(*) FROM cohort_backfill_chunks WHERE run_id = $1 AND status = 'pending'",
)
.bind(seeding_run)
.fetch_one(&pool)
.await?;
ensure!(
still_pending == 1,
"a claimable chunk remained, so the run status is what blocked the claim"
);
ensure_lease_lost(test_support::heartbeat(&store, live_lease, &claimant, lease60).await)?;
drop(claimed);
// Idempotent: a later pass re-reads the same exhausted chunks, and the already-terminal run
// must not be counted again.
ensure!(
fail_exhausted_runs_of_kind(
&pool,
&store,
&run_ids,
RunKind::Behavioral,
attempts5
)
.await
== 0
);
ensure!(matches!(
fail_run(&pool, seeding_run, &RenderedError::from_message("again")).await,
Err(RunError::NotActive(_))
));
Ok(())
})
.await
}
/// Both persisted error columns are truncated to the limit: `chunk.fail` clamps `last_error` and
/// `fail_run` clamps `run.error`, each flipping the row to `failed`.
#[tokio::test]
async fn fail_truncates_chunk_and_run_error_columns() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(
planned_count(
store
.plan_chunks(seeding_run, historical([100]), ONE_BAND)
.await?
)? == 1
);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let claimed = store
.claim_next(
&idle(&run_ids),
&Claimant::new("worker-a")?,
lease60,
attempts5,
)
.await?
.context("claimant found no chunk")?;
let lease = claimed.chunk.spec().lease;
let chunk_id = lease.chunk_id();
store
.fail(
lease,
&RenderedError::from_message("x".repeat(5_000)),
NO_BACKOFF,
)
.await?;
drop(claimed);
let (failed_status, error_length): (String, i32) = sqlx::query_as(
"SELECT status, length(last_error)::integer FROM cohort_backfill_chunks WHERE id = $1",
)
.bind(chunk_id)
.fetch_one(&pool)
.await?;
ensure!(failed_status == "failed");
ensure!(error_length == 4_096);
let active_run = insert_run(
&pool,
3,
"team_enablement",
"awaiting_boundary",
false,
empty_pinned(),
)
.await?;
fail_run(
&pool,
active_run,
&RenderedError::from_message("y".repeat(5_000)),
)
.await?;
let (run_status, run_error_length): (String, i32) = sqlx::query_as(
"SELECT status, length(error)::integer FROM cohort_backfill_runs WHERE id = $1",
)
.bind(active_run)
.fetch_one(&pool)
.await?;
ensure!(run_status == RunStatus::Failed.as_str());
ensure!(run_error_length == 4_096);
Ok(())
})
.await
}
/// The dark-by-default proof: discovery with `kinds = [Behavioral]` (the gate-off binding) never
/// yields a person run, while widening the kind set surfaces it with its kind and pinned
/// `person_scan_since` decoded, and the pinned load validates the person payload.
#[tokio::test]
async fn discovery_is_kind_gated_and_person_pinned_load_validates() -> Result<()> {
with_db(|pool| async move {
let behavioral_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let person_run = insert_person_run(
&pool,
2,
"seeding",
true,
person_pinned(&[(10, ACTIVE_HASH), (11, SUPERSEDED_HASH)]),
)
.await?;
insert_participation(
&pool,
person_run,
2,
10,
false,
person_filter(ACTIVE_HASH, "email"),
)
.await?;
insert_participation(
&pool,
person_run,
2,
11,
true,
person_filter(SUPERSEDED_HASH, "plan"),
)
.await?;
let behavioral_only =
discover_runs(&pool, &TeamAllowlist::All, &[RunKind::Behavioral]).await?;
ensure!(behavioral_only.iter().all(|run| run.run_id != person_run));
ensure!(behavioral_only
.iter()
.any(|run| run.run_id == behavioral_run));
let both_kinds = discover_runs(
&pool,
&TeamAllowlist::All,
&[RunKind::Behavioral, RunKind::PersonProperty],
)
.await?;
let discovered = both_kinds
.iter()
.find(|run| run.run_id == person_run)
.cloned()
.context("person run was not discovered with the widened kind set")?;
ensure!(discovered.kind == RunKind::PersonProperty);
ensure!(discovered.person_scan_since.is_some());
let behavioral = both_kinds
.iter()
.find(|run| run.run_id == behavioral_run)
.context("behavioral run missing from the widened discovery")?;
ensure!(behavioral.kind == RunKind::Behavioral);
let seedable = match establish_boundary(&pool, discovered).await? {
BoundaryOutcome::Established(run) | BoundaryOutcome::AlreadyEstablished(run) => run,
BoundaryOutcome::NoLongerSeedable { .. } => bail!("person run was not seedable"),
};
let PersonRunValidation::Seedable(validated) = seedable.load_person_pinned(&pool).await?
else {
bail!("person run with an active participation unexpectedly retired");
};
ensure!(validated.run.conditions.len() == 1);
ensure!(validated.run.horizon_days == 30);
ensure!(validated.warnings.iter().any(|warning| matches!(
warning,
PinnedWarning::ConditionSuperseded { cohort_id, .. } if *cohort_id == CohortId(11)
)));
Ok(())
})
.await
}
/// The cluster-wide planning claim is exclusive per run and frees on release, so at most one
/// replica ever runs a run's boundary scan.
#[tokio::test]
async fn person_planning_claim_is_exclusive_per_run_until_released() -> Result<()> {
with_db(|pool| async move {
let run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
let other_run = insert_person_run(&pool, 3, "seeding", true, person_pinned(&[])).await?;
let store = PgChunkStore::new(pool.clone());
let held = store
.try_claim_person_planning(run)
.await?
.context("first claim should win")?;
ensure!(store.try_claim_person_planning(run).await?.is_none());
// Claims are per run: a different run's planner is unaffected.
let other = store
.try_claim_person_planning(other_run)
.await?
.context("a different run's claim should win")?;
held.release().await;
other.release().await;
let reclaimed = store
.try_claim_person_planning(run)
.await?
.context("a released claim should be reclaimable")?;
reclaimed.release().await;
Ok(())
})
.await
}
/// Person planning is all-or-nothing: one racing planner wins and inserts every range under the
/// sentinel day with ordinal bands, the loser and any re-plan report `AlreadyPlanned`, and a
/// non-seeding run refuses.
#[tokio::test]
async fn person_planning_is_all_or_nothing_and_idempotent() -> Result<()> {
with_db(|pool| async move {
let run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
let store = PgChunkStore::new(pool.clone());
let boundaries = [Uuid::from_u128(0x10), Uuid::from_u128(0x20)];
let ranges = tile_ranges(&boundaries)?;
ensure!(ranges.len() == 3);
let (left, right) = tokio::join!(
store.plan_person_chunks(run, &ranges),
store.plan_person_chunks(run, &ranges),
);
let outcomes = [left?, right?];
ensure!(
outcomes
.iter()
.filter(|outcome| matches!(outcome, PlanOutcome::Planned { inserted: 3 }))
.count()
== 1,
"expected exactly one winning planner, got {outcomes:?}"
);
ensure!(
outcomes
.iter()
.filter(|outcome| matches!(outcome, PlanOutcome::AlreadyPlanned))
.count()
== 1,
"expected the losing planner to observe AlreadyPlanned, got {outcomes:?}"
);
ensure!(matches!(
store.plan_person_chunks(run, &ranges).await?,
PlanOutcome::AlreadyPlanned
));
let rows: Vec<(String, i16, Option<Uuid>, Option<Uuid>)> = sqlx::query_as(
"SELECT day::text, band, person_range_lo, person_range_hi \
FROM cohort_backfill_chunks WHERE run_id = $1 ORDER BY band",
)
.bind(run)
.fetch_all(&pool)
.await?;
ensure!(rows.len() == 3);
ensure!(rows.iter().all(|(day, ..)| day == "9999-01-01"));
ensure!(rows.iter().map(|(_, band, ..)| *band).eq(0..3));
ensure!(rows[0].2 == Some(Uuid::nil()));
ensure!(rows[0].3 == Some(boundaries[0]));
ensure!(rows[1].2 == Some(boundaries[0]));
ensure!(rows[1].3 == Some(boundaries[1]));
ensure!(rows[2].2 == Some(boundaries[1]));
ensure!(rows[2].3.is_none());
let idle =
insert_person_run(&pool, 3, "awaiting_boundary", false, person_pinned(&[])).await?;
ensure!(matches!(
store.plan_person_chunks(idle, &ranges).await?,
PlanOutcome::RunNotSeeding
));
Ok(())
})
.await
}
/// Behavioral days claim newest first across runs with equal slot loads, and person chunks claim
/// strictly after every behavioral day on the shared claim loop, even though their sentinel day is
/// the newest and their run holds fewer slots. A person claim decodes its range with `num_bands` =
/// the run's chunk count.
#[tokio::test]
async fn behavioral_days_claim_newest_first_then_person_chunks_with_ranges() -> Result<()> {
with_db(|pool| async move {
let behavioral_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
// Another team, because a team holds one active team-scoped run per kind.
let other_team_run =
insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
let person_run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(
planned_count(
store
.plan_chunks(behavioral_run, historical([98, 100]), ONE_BAND)
.await?
)? == 2
);
ensure!(
planned_count(
store
.plan_chunks(other_team_run, historical([99]), ONE_BAND)
.await?
)? == 1
);
let boundary = Uuid::from_u128(0x42);
let ranges = tile_ranges(&[boundary])?;
ensure!(planned_count(store.plan_person_chunks(person_run, &ranges).await?)? == 2);
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let busy = SlotLoad { team: 1, run: 1 };
let candidates = ClaimCandidates::new(vec![
ClaimCandidate {
run_id: behavioral_run,
load: busy,
},
ClaimCandidate {
run_id: other_team_run,
load: busy,
},
ClaimCandidate {
run_id: person_run,
load: SlotLoad::default(),
},
])
.context("no claim candidates")?;
let claimant = Claimant::new("worker-a")?;
let mut behavioral_claims = Vec::new();
for _ in 0..3 {
let claim = store
.claim_next(&candidates, &claimant, lease60, attempts5)
.await?
.context("behavioral chunk was not claimed before person chunks")?;
ensure!(claim.chunk.spec().person_range.is_none());
behavioral_claims.push((claim.chunk.spec().lease.run_id(), claim.chunk.spec().day));
}
ensure!(
behavioral_claims
== [
(behavioral_run, 100),
(other_team_run, 99),
(behavioral_run, 98)
],
"expected newest day first across runs, got {behavioral_claims:?}"
);
let first_person = store
.claim_next(&candidates, &claimant, lease60, attempts5)
.await?
.context("first person chunk was not claimed")?;
let first_person_spec = first_person.chunk.spec();
ensure!(first_person_spec.lease.run_id() == person_run);
ensure!(first_person_spec.band.band() == 0);
ensure!(first_person_spec.band.num_bands().get() == 2);
let first_person_range = first_person_spec
.person_range
.context("person claim lost its range")?;
ensure!(first_person_range.lo() == Uuid::nil());
ensure!(first_person_range.hi() == Some(boundary));
let second_person = store
.claim_next(&candidates, &claimant, lease60, attempts5)
.await?
.context("second person chunk was not claimed")?;
let second_person_range = second_person
.chunk
.spec()
.person_range
.context("person claim lost its range")?;
ensure!(second_person_range.lo() == boundary);
ensure!(second_person_range.hi().is_none());
drop((first_person, second_person));
Ok(())
})
.await
}
/// A claim takes a lapsed trailing day first, then serves the team holding the fewest slots, then
/// the run holding the fewest, and only then the newest day.
#[tokio::test]
async fn claims_serve_the_least_loaded_team_then_run_before_the_newest_day() -> Result<()> {
with_db(|pool| async move {
let heavy_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
// A team holds one active team-scoped run per kind, so the first run moves to cohort
// scope before the second is inserted.
sqlx::query(
"UPDATE cohort_backfill_runs SET scope = 'cohort', cohort_id = 1 WHERE id = $1",
)
.bind(heavy_run)
.execute(&pool)
.await?;
let sibling_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let other_team_run =
insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
let trailing_run =
insert_run(&pool, 4, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
for (run_id, day) in [(heavy_run, 100), (sibling_run, 98), (other_team_run, 97)] {
ensure!(
planned_count(
store
.plan_chunks(run_id, historical([day]), ONE_BAND)
.await?
)? == 1
);
}
let lapsed = Utc::now() - chrono::Duration::hours(1);
ensure!(
planned_count(
store
.plan_chunks(trailing_run, [trailing(96, lapsed)], ONE_BAND)
.await?
)? == 1
);
sqlx::query("UPDATE cohort_backfill_runs SET status = 'trailing' WHERE id = $1")
.bind(trailing_run)
.execute(&pool)
.await?;
let lease60 = LeaseDuration::new(Duration::from_secs(60))?;
let attempts5 = MaxAttempts::new(5)?;
let claimant = Claimant::new("worker-a")?;
let rounds = [
// Team 4 holds the most slots and its trailing day is the oldest day, but the trailing
// day still goes first.
([(2, 2), (2, 0), (1, 1), (3, 0)], (trailing_run, 96)),
// Team 2 holds two slots and team 3 one, so team 3 goes first, ahead of team 2's newer
// days and of its sibling run that holds none.
([(2, 2), (2, 0), (1, 1), (4, 1)], (other_team_run, 97)),
// Both teams hold one slot, so inside team 2 the run holding none goes first, ahead of
// the heavy run's newer day.
([(1, 1), (1, 0), (1, 1), (4, 1)], (sibling_run, 98)),
];
for (loads, expected) in rounds {
let candidates = [heavy_run, sibling_run, other_team_run, trailing_run]
.into_iter()
.zip(loads)
.map(|(run_id, (team, run))| ClaimCandidate {
run_id,
load: SlotLoad { team, run },
})
.collect();
let candidates = ClaimCandidates::new(candidates).context("no claim candidates")?;
let claim = store
.claim_next(&candidates, &claimant, lease60, attempts5)
.await?
.context("no chunk was claimed")?;
let claimed = (claim.chunk.spec().lease.run_id(), claim.chunk.spec().day);
ensure!(
claimed == expected,
"expected {expected:?}, got {claimed:?}"
);
}
Ok(())
})
.await
}
/// Cancelling a run breaks its live leases: the claimed chunk's next heartbeat fences out, so the
/// lease surfaces `Heartbeat(LeaseLost)` and its cancellation token trips.
#[tokio::test]
async fn cancelling_a_run_kills_its_live_lease_via_the_heartbeat() -> Result<()> {
with_db(|pool| async move {
let seeding_run =
insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
let store = PgChunkStore::new(pool.clone());
ensure!(planned_count(store.plan_chunks(seeding_run, historical([100]), ONE_BAND).await?)? == 1);
let lease3 = LeaseDuration::new(Duration::from_secs(3))?;
let attempts5 = MaxAttempts::new(5)?;
let run_ids = [seeding_run];
let observed = store
.claim_next(&idle(&run_ids), &Claimant::new("worker-f")?, lease3, attempts5)
.await?
.context("claimant found no chunk")?;
let cancelled = observed.lease.cancellation_token();
sqlx::query(
"UPDATE cohort_backfill_runs SET status = 'cancelled', updated_at = now() WHERE id = $1",
)
.bind(seeding_run)
.execute(&pool)
.await?;
let failure =
tokio::time::timeout(Duration::from_secs(3), observed.lease.failure()).await?;
ensure!(matches!(
failure,
LeaseFailure::Heartbeat(ChunkStoreError::LeaseLost { .. })
));
ensure!(cancelled.is_cancelled());
Ok(())
})
.await
}