Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

45.7 KB
Raw Blame
//! PostgreSQL contract for the backfill-completion protocol, split into independent scenarios.
//!
//! Each `#[tokio::test]` connects its own schema-scoped database ([`support::with_db`]) and builds
//! only the minimal run/participation state its invariant needs. Together they pin the planning-proof
//! stamp, the `seeding → reconciling` CAS, the atomic dispatch record and reset, the epoch fence on
//! every observation write, the bitmap OR-merge, phase discovery, and the current-hash read.

#![cfg(feature = "pg-test-support")]

use std::collections::BTreeMap;
use std::num::NonZeroU16;
use std::time::Duration;

use anyhow::{ensure, Context, Result};
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use cohort_core::filters::{CohortId, TeamId};
use cohort_core::partitioner::COHORT_PARTITION_COUNT;
use cohort_seeder::domain::{
    CatalogMiss, CompletionPhase, CoverageStartMs, DispatchEpoch, MarkerPartition, MarkerWatch,
    NextOffset, ObservationEnds, PartitionBitmap, PersonShapeHash, ProducedOffset, ReconcileHwms,
    ReconcileScope, RunBoundaryMs, RunId, SeedPartition, UndispatchedReason, WatchPartition,
    WatchPositions, WithheldCohort, WithheldReason,
};
use cohort_seeder::store::chunks::PgChunkStore;
use cohort_seeder::store::completion::{
    cas_run_reconciling, confirm_reconciling, count_reconciling_by_kind, discover_completions,
    load_current_shape_hashes, load_observation_participations, mark_chunks_planned,
    mark_participation_completed, mark_run_observed, mark_run_observed_unreconcilable,
    persist_marker_observations, persist_observation_ends, read_planning_stamp,
    record_participation_partial, record_participation_shortfall, runs_with_all_chunks_confirmed,
    CompletionStoreError, CurrentShapeHash, PlanningStampOutcome,
};
use cohort_seeder::store::runs::{load_reconcile_run, RunKind};
use cohort_seeder::store::RenderedError;
use cohort_seeder::test_support;
use common_types::cohort::TeamAllowlist;
use sqlx::types::Json;
use sqlx::PgPool;

mod support;
use support::{
    empty_pinned, ensure_fence_lost, historical, insert_access, insert_cohort,
    insert_participation, insert_person_participation, insert_person_run, insert_reconciling_run,
    insert_run, person_pinned, set_marker_bits, trailing, with_db,
};

const ONE_BAND: NonZeroU16 = NonZeroU16::MIN;
const MARKER_TOPIC: &str = "cohort_reconcile_markers";
/// What discovery binds once `SEEDER_PERSON_SEEDS_ENABLED` is on.
const BOTH_KINDS: &[RunKind] = &[RunKind::Behavioral, RunKind::PersonProperty];

fn full_hwms() -> ReconcileHwms {
    let offsets: BTreeMap<SeedPartition, ProducedOffset> =
        SeedPartition::all(COHORT_PARTITION_COUNT)
            .unwrap()
            .map(|partition| {
                (
                    partition,
                    ProducedOffset::new(1_000 + i64::from(partition.as_u16())),
                )
            })
            .collect();
    ReconcileHwms::new(offsets).unwrap()
}

fn captured_ends() -> ObservationEnds {
    let mut ends = ObservationEnds::new();
    ends.insert(WatchPartition::new(0), NextOffset::from_high_watermark(10));
    ends
}

fn empty_watch() -> MarkerWatch {
    MarkerWatch {
        topic: MARKER_TOPIC.to_string(),
        positions: WatchPositions::new(),
        ends: None,
    }
}

/// Stamping the planning proof applies once, is idempotent, and refuses a run that is not seeding.
#[tokio::test]
async fn planning_stamp_is_idempotent_and_refuses_non_seeding() -> Result<()> {
    with_db(|pool| async move {
        let seeding =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        ensure!(matches!(
            mark_chunks_planned(&pool, seeding, RunKind::Behavioral).await?,
            PlanningStampOutcome::Stamped
        ));
        ensure!(matches!(
            mark_chunks_planned(&pool, seeding, RunKind::Behavioral).await?,
            PlanningStampOutcome::Skipped
        ));

        let reconciling = insert_reconciling_run(&pool, 3).await?;
        ensure!(matches!(
            mark_chunks_planned(&pool, reconciling, RunKind::Behavioral).await?,
            PlanningStampOutcome::Skipped
        ));
        Ok(())
    })
    .await
}

/// Discovery is gated by the caller's allowed kinds — the dark proof: with the person gate off a
/// person run is invisible, and with it on the same run surfaces carrying its kind.
#[tokio::test]
async fn completion_discovery_is_gated_by_the_allowed_kinds() -> Result<()> {
    with_db(|pool| async move {
        let person_run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;

        let dark = discover_completions(
            &pool,
            &TeamAllowlist::All,
            &[RunKind::Behavioral],
            MARKER_TOPIC,
        )
        .await?;
        ensure!(dark
            .iter()
            .all(|completion| completion.run_id != person_run));

        let lit =
            discover_completions(&pool, &TeamAllowlist::All, BOTH_KINDS, MARKER_TOPIC).await?;
        let discovered = lit
            .iter()
            .find(|completion| completion.run_id == person_run)
            .context("the person run surfaces once its kind is allowed")?;
        ensure!(discovered.kind == RunKind::PersonProperty);
        Ok(())
    })
    .await
}

/// Every kind-bound entry point refuses a run id presented under the other kind, so the two
/// protocols can never write each other's rows.
#[tokio::test]
async fn kind_bound_entry_points_refuse_the_other_kind() -> Result<()> {
    with_db(|pool| async move {
        let person_run = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
        let behavioral_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;

        ensure!(matches!(
            mark_chunks_planned(&pool, person_run, RunKind::Behavioral).await?,
            PlanningStampOutcome::Skipped
        ));
        ensure!(matches!(
            mark_chunks_planned(&pool, person_run, RunKind::PersonProperty).await?,
            PlanningStampOutcome::Stamped
        ));
        ensure!(
            read_planning_stamp(&pool, person_run, RunKind::PersonProperty)
                .await?
                .is_some()
        );
        ensure!(read_planning_stamp(&pool, person_run, RunKind::Behavioral)
            .await?
            .is_none());
        mark_chunks_planned(&pool, behavioral_run, RunKind::Behavioral).await?;

        // The CAS out of `seeding` is the gate into the reconcile protocol; each kind holds only
        // its own runs.
        ensure!(cas_run_reconciling(&pool, person_run, RunKind::Behavioral)
            .await?
            .is_none());
        ensure!(
            cas_run_reconciling(&pool, behavioral_run, RunKind::PersonProperty)
                .await?
                .is_none()
        );
        ensure!(
            cas_run_reconciling(&pool, person_run, RunKind::PersonProperty)
                .await?
                .is_some()
        );

        // And once reconciling, the re-dispatch claim is bound the same way.
        ensure!(confirm_reconciling(&pool, person_run, RunKind::Behavioral)
            .await?
            .is_none());
        ensure!(
            confirm_reconciling(&pool, person_run, RunKind::PersonProperty)
                .await?
                .is_some()
        );
        Ok(())
    })
    .await
}

/// A person run's fence is its person hash: `load_reconcile_run` and the observation load both read
/// the kind-matching column and ignore the `''` sibling that would otherwise fail closed.
#[tokio::test]
async fn person_run_loads_pin_the_person_hash_and_ignore_the_empty_behavioral_column() -> Result<()>
{
    with_db(|pool| async move {
        let run_id = insert_person_run(&pool, 2, "seeding", true, person_pinned(&[])).await?;
        insert_person_participation(&pool, run_id, 2, 301, "person-shape").await?;

        let run = load_reconcile_run(&pool, run_id).await?;
        ensure!(run.kind() == RunKind::PersonProperty);
        let boundary_at: DateTime<Utc> =
            sqlx::query_scalar("SELECT boundary_at FROM cohort_backfill_runs WHERE id = $1")
                .bind(run_id)
                .fetch_one(&pool)
                .await?;
        ensure!(run
            .tiles()
            .all(|tile| tile.boundary() == Some(RunBoundaryMs(boundary_at.timestamp_millis()))));
        let scopes: Vec<ReconcileScope> = run.tiles().map(|tile| tile.scope().clone()).collect();
        ensure!(
            scopes
                == vec![ReconcileScope::PersonProperty(PersonShapeHash::parse(
                    "person-shape"
                )?)]
        );

        let participations =
            load_observation_participations(&pool, run_id, RunKind::PersonProperty).await?;
        ensure!(participations.len() == 1);
        ensure!(participations[0].scope == scopes[0]);

        // Read under the wrong kind and the sibling column's `''` fails closed rather than
        // silently pinning nothing. Symmetric: a behavioral run read as person does the same.
        let behavioral_run =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        insert_participation(&pool, behavioral_run, 2, 302, false, empty_pinned()).await?;
        for (run, kind) in [
            (run_id, RunKind::Behavioral),
            (behavioral_run, RunKind::PersonProperty),
        ] {
            ensure!(matches!(
                load_observation_participations(&pool, run, kind).await,
                Err(CompletionStoreError::InvalidShapeHash { .. })
            ));
        }
        Ok(())
    })
    .await
}

/// The CAS admits exactly one winner, and refuses a run that has not planned or still has an
/// unconfirmed chunk.
#[tokio::test]
async fn cas_reconciling_is_single_winner_and_gated_on_planned_and_confirmed() -> Result<()> {
    with_db(|pool| async move {
        // Planned, no chunks: two racing CAS attempts, exactly one wins.
        let ready =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        mark_chunks_planned(&pool, ready, RunKind::Behavioral).await?;
        let (left, right) = tokio::join!(
            cas_run_reconciling(&pool, ready, RunKind::Behavioral),
            cas_run_reconciling(&pool, ready, RunKind::Behavioral)
        );
        let winners = [left?, right?].into_iter().filter(Option::is_some).count();
        ensure!(
            winners == 1,
            "expected exactly one CAS winner, got {winners}"
        );

        // Unplanned seeding run: CAS refused.
        let unplanned =
            insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
        ensure!(cas_run_reconciling(&pool, unplanned, RunKind::Behavioral)
            .await?
            .is_none());

        // Planned but with an unconfirmed chunk: CAS refused.
        let pending =
            insert_run(&pool, 4, "team_enablement", "seeding", true, empty_pinned()).await?;
        mark_chunks_planned(&pool, pending, RunKind::Behavioral).await?;
        PgChunkStore::new(pool.clone())
            .plan_chunks(pending, historical([100]), ONE_BAND)
            .await?;
        ensure!(cas_run_reconciling(&pool, pending, RunKind::Behavioral)
            .await?
            .is_none());

        // Only a held trailing day unconfirmed: readiness does not wait for it, so the CAS wins.
        let trailing_only =
            insert_run(&pool, 5, "team_enablement", "seeding", true, empty_pinned()).await?;
        mark_chunks_planned(&pool, trailing_only, RunKind::Behavioral).await?;
        let store = PgChunkStore::new(pool.clone());
        store
            .plan_chunks(
                trailing_only,
                [trailing(101, Utc::now() + ChronoDuration::hours(1))],
                ONE_BAND,
            )
            .await?;
        ensure!(store.chunk_progress(trailing_only).await?.remaining() == 0);
        ensure!(
            cas_run_reconciling(&pool, trailing_only, RunKind::Behavioral)
                .await?
                .is_some()
        );
        Ok(())
    })
    .await
}

/// Recording a dispatch writes the HWMs and watch, resets every non-superseded participation's
/// bits/completion/error, nulls the observed stamp, and mints the fence epoch; a re-dispatch stamps a
/// fresh epoch and resets again.
#[tokio::test]
async fn record_dispatch_resets_participations_and_mints_fresh_epoch() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        set_marker_bits(&pool, run_id, 301, 7).await?;
        sqlx::query(
            "UPDATE cohort_backfill_run_cohorts SET reconcile_completed_at = now(), error = 'stale' \
             WHERE run_id = $1 AND cohort_id = 301",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;

        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable for dispatch")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        let (bits, completed, error): (i64, Option<chrono::DateTime<chrono::Utc>>, String) =
            sqlx::query_as(
                "SELECT reconcile_marker_bits, reconcile_completed_at, error \
                 FROM cohort_backfill_run_cohorts WHERE run_id = $1 AND cohort_id = 301",
            )
            .bind(run_id)
            .fetch_one(&pool)
            .await?;
        ensure!(bits == 0 && completed.is_none() && error.is_empty());

        let (hwms_set, observed): (bool, Option<chrono::DateTime<chrono::Utc>>) = sqlx::query_as(
            "SELECT reconcile_hwms IS NOT NULL, reconcile_observed_at \
             FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(hwms_set && observed.is_none());
        ensure!(test_support::dispatch_epoch(&pool, run_id).await? == epoch);

        // Re-dispatch: a fresh claim records a new fence at or after the first.
        set_marker_bits(&pool, run_id, 301, 5).await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("re-dispatch should be claimable")?;
        let next_epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;
        ensure!(next_epoch.as_datetime() >= epoch.as_datetime());
        ensure!(test_support::dispatch_epoch(&pool, run_id).await? == next_epoch);
        let bits: i64 = sqlx::query_scalar(
            "SELECT reconcile_marker_bits FROM cohort_backfill_run_cohorts \
             WHERE run_id = $1 AND cohort_id = 301",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(bits == 0);
        Ok(())
    })
    .await
}

/// Reverting a claim returns the run to seeding and clears any dispatch record, so it re-enters
/// discovery as a plain planned seeding run instead of the seeding-with-reconcile-columns anomaly.
#[tokio::test]
async fn revert_clears_the_dispatch_record_and_returns_to_seeding() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let _ = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be re-claimable")?;
        claim.revert(&pool).await?;

        let (status, cleared): (String, bool) = sqlx::query_as(
            "SELECT status, reconcile_dispatched_at IS NULL AND reconcile_observed_at IS NULL \
             AND reconcile_hwms IS NULL AND marker_watch IS NULL \
             FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(status == "seeding" && cleared);

        let discovered =
            discover_completions(&pool, &TeamAllowlist::All, BOTH_KINDS, MARKER_TOPIC).await?;
        let phase = discovered
            .iter()
            .find(|completion| completion.run_id == run_id)
            .map(|completion| completion.phase.clone());
        ensure!(phase == Some(CompletionPhase::SeedingPlanned));
        Ok(())
    })
    .await
}

/// A claim minted before another dispatcher recorded its dispatch cannot revert it — two replicas
/// can hold a claim for one run, and an unfenced revert would bounce a live dispatch to `seeding`.
#[tokio::test]
async fn revert_is_fenced_against_a_dispatch_recorded_after_the_claim() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        let stale = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let winner = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable twice")?;
        let epoch = winner.record(&pool, &full_hwms(), &empty_watch()).await?;

        stale.revert(&pool).await?;

        let (status, dispatched_at): (String, Option<DateTime<Utc>>) = sqlx::query_as(
            "SELECT status, reconcile_dispatched_at FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(status == "reconciling", "revert bounced a live dispatch");
        ensure!(dispatched_at == Some(epoch.as_datetime()));
        mark_run_observed(&pool, run_id, epoch).await?;
        Ok(())
    })
    .await
}

/// A claim that lost the race cannot record over the winner's dispatch. Recording resets marker bits
/// and completions, so an unfenced second write would discard observations already merged under the
/// ruling epoch.
#[tokio::test]
async fn record_is_fenced_against_a_dispatch_recorded_after_the_claim() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        let stale = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let winner = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable twice")?;
        let epoch = winner.record(&pool, &full_hwms(), &empty_watch()).await?;
        persist_marker_observations(
            &pool,
            run_id,
            epoch,
            &[(CohortId(301), PartitionBitmap::from_bits(3)?)],
            &[],
            &WatchPositions::new(),
            MARKER_TOPIC,
        )
        .await?;

        ensure_fence_lost(
            stale
                .record(&pool, &full_hwms(), &empty_watch())
                .await
                .map(|_| ()),
        )?;

        let bits: i64 = sqlx::query_scalar(
            "SELECT reconcile_marker_bits FROM cohort_backfill_run_cohorts \
             WHERE run_id = $1 AND cohort_id = 301",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(
            bits == 3,
            "a lost claim reset the ruling epoch's marker bits"
        );
        ensure!(test_support::dispatch_epoch(&pool, run_id).await? == epoch);
        Ok(())
    })
    .await
}

/// The batched ledger check returns exactly the runs whose chunks have all confirmed.
#[tokio::test]
async fn runs_with_all_chunks_confirmed_selects_only_fully_confirmed_ledgers() -> Result<()> {
    with_db(|pool| async move {
        let store = PgChunkStore::new(pool.clone());
        let no_chunks =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;
        let confirmed =
            insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
        store
            .plan_chunks(confirmed, historical([100]), ONE_BAND)
            .await?;
        sqlx::query("UPDATE cohort_backfill_chunks SET status = 'confirmed' WHERE run_id = $1")
            .bind(confirmed)
            .execute(&pool)
            .await?;
        let pending =
            insert_run(&pool, 4, "team_enablement", "seeding", true, empty_pinned()).await?;
        store
            .plan_chunks(pending, historical([100, 101]), ONE_BAND)
            .await?;
        sqlx::query(
            "UPDATE cohort_backfill_chunks SET status = 'confirmed' \
             WHERE id = (SELECT id FROM cohort_backfill_chunks WHERE run_id = $1 LIMIT 1)",
        )
        .bind(pending)
        .execute(&pool)
        .await?;

        let trailing_only =
            insert_run(&pool, 5, "team_enablement", "seeding", true, empty_pinned()).await?;
        store
            .plan_chunks(
                trailing_only,
                [trailing(101, Utc::now() + ChronoDuration::hours(1))],
                ONE_BAND,
            )
            .await?;

        let ready =
            runs_with_all_chunks_confirmed(&pool, &[no_chunks, confirmed, pending, trailing_only])
                .await?;
        ensure!(
            ready == [no_chunks, confirmed, trailing_only].into_iter().collect(),
            "unexpected ready set: {ready:?}"
        );
        ensure!(runs_with_all_chunks_confirmed(&pool, &[]).await?.is_empty());
        Ok(())
    })
    .await
}

/// A participation Django already superseded keeps the reason Django recorded for it.
#[tokio::test]
async fn recording_a_partial_preserves_an_existing_supersession_reason() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, true, empty_pinned()).await?;
        sqlx::query(
            "UPDATE cohort_backfill_run_cohorts SET error = 'Cohort definition changed during backfill' \
             WHERE run_id = $1",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        record_participation_partial(
            &pool,
            run_id,
            epoch,
            CohortId(301),
            &RenderedError::from_message("markers short; cohort diverged"),
        )
        .await?;

        let error: String = sqlx::query_scalar(
            "SELECT error FROM cohort_backfill_run_cohorts WHERE run_id = $1 AND cohort_id = 301",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(
            error == "Cohort definition changed during backfill",
            "reconcile outcome overwrote the original supersession reason: {error}"
        );
        Ok(())
    })
    .await
}

/// A withheld supersede rides the fenced flush and spares completed, stamped and superseded rows.
#[tokio::test]
async fn a_withheld_supersede_rides_the_fenced_marker_flush() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        for cohort_id in [401, 402, 403, 404] {
            insert_participation(&pool, run_id, 2, cohort_id, false, empty_pinned()).await?;
        }
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;
        mark_participation_completed(&pool, run_id, epoch, CohortId(402)).await?;
        sqlx::query(
            "UPDATE cohort_backfill_run_cohorts SET stamped_at = now() \
             WHERE run_id = $1 AND cohort_id = 403",
        )
        .bind(run_id)
        .execute(&pool)
        .await?;
        let edited = RenderedError::from_message("Cohort definition changed during backfill");
        record_participation_partial(&pool, run_id, epoch, CohortId(404), &edited).await?;

        let withheld = |cohort_id: i32, reason: WithheldReason| WithheldCohort {
            cohort_id: CohortId(cohort_id),
            partition: MarkerPartition::new(7).expect("7 is a marker partition"),
            reason,
        };
        let catalog = WithheldReason::Catalog(CatalogMiss::HashUnknown);
        let verdicts = [
            withheld(
                401,
                WithheldReason::PartialCoverage {
                    covered_since: CoverageStartMs(1_791_547_200_000),
                },
            ),
            withheld(402, catalog),
            withheld(403, catalog),
            withheld(404, catalog),
        ];
        let mut bits = PartitionBitmap::default();
        bits.set(MarkerPartition::new(0)?);
        let bit_updates = [(CohortId(401), bits)];
        let mut advanced = WatchPositions::new();
        advanced.insert(WatchPartition::new(0), NextOffset::from_high_watermark(42));

        let stale = test_support::epoch_at(epoch.as_datetime() - ChronoDuration::hours(1));
        ensure_fence_lost(
            persist_marker_observations(
                &pool,
                run_id,
                stale,
                &bit_updates,
                &verdicts,
                &advanced,
                MARKER_TOPIC,
            )
            .await,
        )?;
        let outcomes = || async {
            sqlx::query_as::<_, (i32, bool, String, i64)>(
                "SELECT cohort_id, superseded_at IS NOT NULL, error, reconcile_marker_bits \
                 FROM cohort_backfill_run_cohorts WHERE run_id = $1 ORDER BY cohort_id",
            )
            .bind(run_id)
            .fetch_all(&pool)
            .await
        };
        let positions = || async {
            sqlx::query_scalar::<_, Json<MarkerWatch>>(
                "SELECT marker_watch FROM cohort_backfill_runs WHERE id = $1",
            )
            .bind(run_id)
            .fetch_one(&pool)
            .await
            .map(|watch| watch.0.positions)
        };
        let before = outcomes().await?;
        ensure!(
            before.iter().map(|row| row.1).collect::<Vec<_>>() == [false, false, false, true],
            "a stale epoch superseded a participation: {before:?}"
        );
        ensure!(before[0].3 == 0, "a stale epoch merged bits");
        ensure!(positions().await? == WatchPositions::new());

        persist_marker_observations(
            &pool,
            run_id,
            epoch,
            &bit_updates,
            &verdicts,
            &advanced,
            MARKER_TOPIC,
        )
        .await?;

        let after = outcomes().await?;
        ensure!(
            after.iter().map(|row| row.1).collect::<Vec<_>>() == [true, false, false, true],
            "only the open participation is superseded: {after:?}"
        );
        ensure!(
            after[0].2.contains("partition 7 (partial_coverage)")
                && after[0].2.contains("2026-10-09T12:00:00+00:00"),
            "the error must name the partition and the recovery instant: {}",
            after[0].2
        );
        ensure!(after[0].3 != 0, "the bits rode the same flush");
        ensure!(after[3].2 == edited.as_str(), "an earlier reason is kept");
        ensure!(positions().await? == advanced);
        Ok(())
    })
    .await
}

/// Every fenced observation write is rejected under a stale epoch and under a run whose status has
/// left `reconciling`, while a matching epoch on a reconciling run is accepted.
#[tokio::test]
async fn fenced_writes_reject_stale_epoch_and_wrong_status() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        // A matching epoch on a reconciling run is accepted.
        mark_participation_completed(&pool, run_id, epoch, CohortId(301)).await?;

        let stale = test_support::epoch_at(epoch.as_datetime() - ChronoDuration::hours(1));
        let bit_updates = [(CohortId(301), PartitionBitmap::default())];
        let error = RenderedError::from_message("shortfall");
        for at in [stale, epoch] {
            if at == epoch {
                // Same valid epoch, but flip the run out of reconciling: the status guard fences.
                sqlx::query("UPDATE cohort_backfill_runs SET status = 'completed' WHERE id = $1")
                    .bind(run_id)
                    .execute(&pool)
                    .await?;
            }
            ensure_fence_lost(
                persist_observation_ends(&pool, run_id, at, &captured_ends(), MARKER_TOPIC).await,
            )?;
            ensure_fence_lost(
                persist_marker_observations(
                    &pool,
                    run_id,
                    at,
                    &bit_updates,
                    &[],
                    &WatchPositions::new(),
                    MARKER_TOPIC,
                )
                .await,
            )?;
            ensure_fence_lost(
                mark_participation_completed(&pool, run_id, at, CohortId(301)).await,
            )?;
            ensure_fence_lost(
                record_participation_partial(&pool, run_id, at, CohortId(301), &error).await,
            )?;
            ensure_fence_lost(
                record_participation_shortfall(&pool, run_id, at, CohortId(301), &error).await,
            )?;
            ensure_fence_lost(mark_run_observed(&pool, run_id, at).await)?;
        }
        Ok(())
    })
    .await
}

/// A participation write whose epoch is superseded *while the statement runs* must still lose the
/// fence. The three participation writes fence through a CTE rather than a run-row UPDATE, so
/// without `FOR UPDATE` the CTE would be evaluated against the pre-block snapshot (epoch still
/// current), the participation UPDATE would unblock after the re-dispatch committed, and
/// EvalPlanQual — which re-checks only the participation row's quals — would land the stale write
/// inside the new epoch's regime while reporting success.
#[tokio::test]
async fn fenced_participation_writes_lose_an_epoch_superseded_mid_statement() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let _initial = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        for write in [
            FencedWrite::Completed,
            FencedWrite::Partial,
            FencedWrite::Shortfall,
        ] {
            let current: DateTime<Utc> = sqlx::query_scalar(
                "SELECT reconcile_dispatched_at FROM cohort_backfill_runs WHERE id = $1",
            )
            .bind(run_id)
            .fetch_one(&pool)
            .await?;
            let epoch = test_support::epoch_at(current);

            // A re-dispatch that has stamped the new epoch but not yet committed. Only the run row
            // is touched, so nothing but the fence itself can block the write under test.
            let mut redispatch = pool.begin().await?;
            sqlx::query(
                "UPDATE cohort_backfill_runs \
                 SET reconcile_dispatched_at = reconcile_dispatched_at + interval '1 second' \
                 WHERE id = $1",
            )
            .bind(run_id)
            .execute(&mut *redispatch)
            .await?;

            let mut pending = tokio::spawn(fenced_write(pool.clone(), run_id, epoch, write));
            // The fence must hold the run row's lock, so the write cannot resolve while the
            // re-dispatch transaction is open. An unfenced read would return `Ok` immediately here.
            let early = tokio::time::timeout(Duration::from_millis(500), &mut pending).await;
            ensure!(
                early.is_err(),
                "{write:?} resolved past an in-flight re-dispatch instead of blocking on the fence"
            );

            redispatch.commit().await?;
            ensure_fence_lost(pending.await?)?;
        }

        let completed: Option<DateTime<Utc>> = sqlx::query_scalar(
            "SELECT reconcile_completed_at FROM cohort_backfill_run_cohorts \
             WHERE run_id = $1 AND cohort_id = 301",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(
            completed.is_none(),
            "a stale-epoch write landed inside the ruling epoch"
        );
        Ok(())
    })
    .await
}

#[derive(Debug, Clone, Copy)]
enum FencedWrite {
    Completed,
    Partial,
    Shortfall,
}

async fn fenced_write(
    pool: PgPool,
    run_id: RunId,
    epoch: DispatchEpoch,
    write: FencedWrite,
) -> std::result::Result<(), CompletionStoreError> {
    let error = RenderedError::from_message("stale-epoch write");
    let cohort_id = CohortId(301);
    match write {
        FencedWrite::Completed => {
            mark_participation_completed(&pool, run_id, epoch, cohort_id).await
        }
        FencedWrite::Partial => {
            record_participation_partial(&pool, run_id, epoch, cohort_id, &error).await
        }
        FencedWrite::Shortfall => {
            record_participation_shortfall(&pool, run_id, epoch, cohort_id, &error).await
        }
    }
}

/// Capturing the observation ends patches only the `ends` key. The observer holds discovery-time
/// positions, so writing the whole document would regress the watcher's flushed coverage — and on a
/// topic that then goes idle nothing would repair it, leaving the run unable to ever mint a proof.
#[tokio::test]
async fn capturing_ends_leaves_the_watcher_positions_alone() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 301, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        let mut advanced = WatchPositions::new();
        advanced.insert(WatchPartition::new(0), NextOffset::from_high_watermark(42));
        persist_marker_observations(&pool, run_id, epoch, &[], &[], &advanced, MARKER_TOPIC)
            .await?;
        persist_observation_ends(&pool, run_id, epoch, &captured_ends(), MARKER_TOPIC).await?;

        let watch: Json<MarkerWatch> =
            sqlx::query_scalar("SELECT marker_watch FROM cohort_backfill_runs WHERE id = $1")
                .bind(run_id)
                .fetch_one(&pool)
                .await?;
        ensure!(
            watch.0.positions.get(WatchPartition::new(0))
                == Some(NextOffset::from_high_watermark(42)),
            "capturing the ends regressed the watcher positions"
        );
        ensure!(watch.0.ends == Some(captured_ends()));
        Ok(())
    })
    .await
}

/// A reconciling run whose every participation was superseded has nothing to dispatch and no outcome
/// to write, so the seeder stamps the observation itself and hands the run to Django's finalizer.
/// The guard refuses to shortcut a run that still has an active participation.
#[tokio::test]
async fn unreconcilable_runs_are_marked_observed_for_django() -> Result<()> {
    with_db(|pool| async move {
        let stranded = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, stranded, 2, 301, true, empty_pinned()).await?;
        mark_run_observed_unreconcilable(&pool, stranded, RunKind::Behavioral).await?;

        let observed_stamp: Option<DateTime<Utc>> = sqlx::query_scalar(
            "SELECT reconcile_observed_at FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(stranded)
        .fetch_one(&pool)
        .await?;
        ensure!(observed_stamp.is_some());
        // Observing it is what removes it from the driver's working set: everything left is
        // Django's, and the gauge reads it from the aggregate instead.
        let discovered =
            discover_completions(&pool, &TeamAllowlist::All, BOTH_KINDS, MARKER_TOPIC).await?;
        ensure!(!discovered
            .iter()
            .any(|completion| completion.run_id == stranded));
        let counts = count_reconciling_by_kind(&pool, &TeamAllowlist::All, BOTH_KINDS).await?;
        ensure!(counts.get(&RunKind::Behavioral) == Some(&1));

        let live = insert_reconciling_run(&pool, 3).await?;
        insert_participation(&pool, live, 3, 401, false, empty_pinned()).await?;
        ensure_fence_lost(
            mark_run_observed_unreconcilable(&pool, live, RunKind::Behavioral).await,
        )?;
        let observed: Option<DateTime<Utc>> = sqlx::query_scalar(
            "SELECT reconcile_observed_at FROM cohort_backfill_runs WHERE id = $1",
        )
        .bind(live)
        .fetch_one(&pool)
        .await?;
        ensure!(observed.is_none());
        Ok(())
    })
    .await
}

/// The observed-marker OR-merge accumulates across calls and round-trips through PostgreSQL,
/// including the bit-63 case that stores as a negative BIGINT.
#[tokio::test]
async fn marker_observations_or_merge_round_trip_including_bit_63() -> Result<()> {
    with_db(|pool| async move {
        let run_id = insert_reconciling_run(&pool, 2).await?;
        insert_participation(&pool, run_id, 2, 401, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, run_id, RunKind::Behavioral)
            .await?
            .context("run should be claimable")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        let mut low = PartitionBitmap::default();
        low.set(MarkerPartition::new(0)?);
        persist_marker_observations(
            &pool,
            run_id,
            epoch,
            &[(CohortId(401), low)],
            &[],
            &WatchPositions::new(),
            MARKER_TOPIC,
        )
        .await?;

        let mut high = PartitionBitmap::default();
        high.set(MarkerPartition::new(63)?);
        persist_marker_observations(
            &pool,
            run_id,
            epoch,
            &[(CohortId(401), high)],
            &[],
            &WatchPositions::new(),
            MARKER_TOPIC,
        )
        .await?;

        let stored: i64 = sqlx::query_scalar(
            "SELECT reconcile_marker_bits FROM cohort_backfill_run_cohorts \
             WHERE run_id = $1 AND cohort_id = 401",
        )
        .bind(run_id)
        .fetch_one(&pool)
        .await?;
        ensure!(stored < 0, "bit 63 must store as a negative BIGINT");

        let participations =
            load_observation_participations(&pool, run_id, RunKind::Behavioral).await?;
        let bits = participations
            .iter()
            .find(|participation| participation.cohort_id == CohortId(401))
            .map(|participation| participation.bits)
            .context("cohort 401 not loaded")?;
        let missing = bits.missing();
        ensure!(!missing.contains(&MarkerPartition::new(0)?));
        ensure!(!missing.contains(&MarkerPartition::new(63)?));
        ensure!(missing.contains(&MarkerPartition::new(1)?));
        Ok(())
    })
    .await
}

/// Discovery classifies each completion phase, including a reconciling run whose completion columns
/// are all NULL and one whose persisted watch names a different marker topic — and drops the observed
/// run, whose remaining work is Django's, while the reconciling-run count still reports it.
#[tokio::test]
async fn discovery_classifies_each_phase_and_drops_observed_runs() -> Result<()> {
    with_db(|pool| async move {
        let unplanned =
            insert_run(&pool, 2, "team_enablement", "seeding", true, empty_pinned()).await?;

        let planned =
            insert_run(&pool, 3, "team_enablement", "seeding", true, empty_pinned()).await?;
        mark_chunks_planned(&pool, planned, RunKind::Behavioral).await?;

        let dispatched = insert_reconciling_run(&pool, 4).await?;
        insert_participation(&pool, dispatched, 4, 401, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, dispatched, RunKind::Behavioral)
            .await?
            .context("dispatched run should be claimable")?;
        let _ = claim.record(&pool, &full_hwms(), &empty_watch()).await?;

        // A run that predates the completion columns: reconciling with every one of them NULL.
        let undispatched = insert_run(
            &pool,
            5,
            "team_enablement",
            "reconciling",
            true,
            empty_pinned(),
        )
        .await?;

        let observed = insert_reconciling_run(&pool, 6).await?;
        insert_participation(&pool, observed, 6, 601, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, observed, RunKind::Behavioral)
            .await?
            .context("observed run should be claimable")?;
        let epoch = claim.record(&pool, &full_hwms(), &empty_watch()).await?;
        mark_run_observed(&pool, observed, epoch).await?;

        // A watch recorded against the topic markers used to ride. Its offsets name a different log,
        // so resuming on them would settle the run against positions it never read.
        let moved = insert_reconciling_run(&pool, 7).await?;
        insert_participation(&pool, moved, 7, 701, false, empty_pinned()).await?;
        let claim = confirm_reconciling(&pool, moved, RunKind::Behavioral)
            .await?
            .context("moved run should be claimable")?;
        let _ = claim
            .record(
                &pool,
                &full_hwms(),
                &MarkerWatch {
                    topic: "cohort_membership_changed_shadow".to_string(),
                    ..empty_watch()
                },
            )
            .await?;

        let discovered =
            discover_completions(&pool, &TeamAllowlist::All, BOTH_KINDS, MARKER_TOPIC).await?;
        let phase = |run| {
            discovered
                .iter()
                .find(|completion| completion.run_id == run)
                .map(|completion| completion.phase.clone())
        };
        ensure!(phase(unplanned) == Some(CompletionPhase::SeedingUnplanned));
        ensure!(phase(planned) == Some(CompletionPhase::SeedingPlanned));
        ensure!(matches!(
            phase(dispatched),
            Some(CompletionPhase::Reconciling(_))
        ));
        ensure!(
            phase(undispatched)
                == Some(CompletionPhase::ReconcilingUndispatched(
                    UndispatchedReason::NeverDispatched
                ))
        );
        ensure!(
            phase(moved)
                == Some(CompletionPhase::ReconcilingUndispatched(
                    UndispatchedReason::TopicChanged
                ))
        );
        // Dropped from discovery rather than hydrated and matched away: a run parked here behind
        // Django's readiness gate would otherwise cost two per-partition JSONB decodes every tick.
        ensure!(phase(observed).is_none());

        // Every reconciling run, observed or not, still reaches the gauge.
        let counts = count_reconciling_by_kind(&pool, &TeamAllowlist::All, BOTH_KINDS).await?;
        ensure!(counts.get(&RunKind::Behavioral) == Some(&4));
        ensure!(!counts.contains_key(&RunKind::PersonProperty));

        // Both statements have a team-scoped variant, and dev — where this rollout lands first —
        // is the environment that runs allowlisted. Both also admit a team the access record grants.
        let granted = insert_reconciling_run(&pool, 8).await?;
        insert_access(&pool, 8, "processing", Duration::from_secs(600)).await?;
        let allowlisted = TeamAllowlist::Only([4, 6].into_iter().collect());
        let scoped = discover_completions(&pool, &allowlisted, BOTH_KINDS, MARKER_TOPIC).await?;
        ensure!(scoped
            .iter()
            .map(|row| row.run_id)
            .eq([dispatched, granted]));
        let scoped_counts = count_reconciling_by_kind(&pool, &allowlisted, BOTH_KINDS).await?;
        ensure!(scoped_counts.get(&RunKind::Behavioral) == Some(&3));
        Ok(())
    })
    .await
}

/// The current-shape read distinguishes present, deleted, absent, and NULL/unparseable cohorts, and
/// reads each kind out of its own column.
#[tokio::test]
async fn current_shape_hashes_read_present_deleted_absent_and_null_per_kind() -> Result<()> {
    with_db(|pool| async move {
        // Distinct per column, so reading the wrong one is a mismatch rather than a coincidence.
        insert_cohort(&pool, 501, 2, Some("behavioral-a"), Some("person-a"), false).await?;
        insert_cohort(&pool, 502, 2, Some("behavioral-b"), Some("person-b"), true).await?;
        insert_cohort(&pool, 503, 2, None, None, false).await?;
        // 504 intentionally absent.

        for (kind, expected) in [
            (RunKind::Behavioral, "behavioral-a"),
            (RunKind::PersonProperty, "person-a"),
        ] {
            let hashes = load_current_shape_hashes(
                &pool,
                TeamId(2),
                &[CohortId(501), CohortId(502), CohortId(503), CohortId(504)],
                kind,
            )
            .await?;

            ensure!(
                hashes.get(&CohortId(501))
                    == Some(&CurrentShapeHash::Present(ReconcileScope::parse(
                        kind.scope(),
                        expected
                    )?)),
                "{kind:?} reads its own column",
            );
            ensure!(hashes.get(&CohortId(502)) == Some(&CurrentShapeHash::Deleted));
            ensure!(hashes.get(&CohortId(503)) == Some(&CurrentShapeHash::Indeterminate));
            ensure!(!hashes.contains_key(&CohortId(504)));
        }
        Ok(())
    })
    .await
}