Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.9 KB
Raw Blame
//! App layer: run discovery → boundary → validate → plan, per run. Depends on `store`, `domain`, and
//! `config`'s allowlist type; never imported by a lower layer.
//!
//! [`refresh_runs`] is the thin fold over discovered runs; [`prepare_run`] dispatches each on its
//! kind — behavioral runs plan inline (cheap day arithmetic), person runs surface a
//! [`PersonPlanRequest`] for the orchestrator's spawned planning slot, since their planning needs a
//! ClickHouse scan that must not run inside the poll loop's liveness deadline. Every behavioral
//! counter and gauge stays at its former point.

use std::collections::{HashMap, HashSet};
use std::sync::Arc;

use cohort_core::bucket_tz::window_start_for_now;
use cohort_core::filters::TeamId;
use common_types::cohort::TeamAllowlist;
use metrics::{counter, gauge};
use sqlx::PgPool;
use tracing::{debug, info, warn};

use crate::domain::{
    plan_days, ConditionAnalyses, ConditionClass, Lookback, PersonEmissionPolicy,
    PersonRunValidation, PinnedError, PinnedPersonRun, PinnedRun, PinnedWarning, PlanCaps,
    ProjectedKeys, RunId,
};
use crate::observability::metrics::{
    team_label, BOUNDARY_CAS_LOST, BOUNDARY_ESTABLISHED, CHUNKS_PLANNED, CONDITIONS_CLASSIFIED,
    CONDITIONS_DROPPED, CONDITIONS_UNANALYZABLE, LOOKBACK_TRUNCATED, RUNS_DISCOVERED,
    RUNS_PLANNING_STAMPED, RUNS_WAITING_BOUNDARY, RUNS_WITHOUT_CHUNKS, RUN_CHUNKS_REMAINING,
    RUN_VALIDATION_FAILURES, TZ_FALLBACK, WINDOW_DAYS_MISMATCH,
};
use crate::store::chunks::{PgChunkStore, PlanOutcome};
use crate::store::completion::{mark_chunks_planned, read_planning_stamp, PlanningStampOutcome};
use crate::store::runs::{
    discover_runs, establish_boundary, fail_run, record_run_warning, BoundaryOutcome,
    DiscoveredRun, RunError, RunKind, RunStatus, RunWarningNote, SeedPhase, SeedableRun,
};
use crate::store::RenderedError;

use super::person_plan::PersonPlanRequest;

/// A run validated to the point of claim eligibility, by kind.
pub(super) enum PreparedRun {
    Behavioral(Arc<PreparedBehavioral>),
    Person(Arc<PinnedPersonRun>),
}

impl PreparedRun {
    /// The claim budget and [`run_ids_of_kind`] both resolve a run's kind through here, so the
    /// mapping lives in one place.
    pub(super) fn kind(&self) -> RunKind {
        match self {
            Self::Behavioral(_) => RunKind::Behavioral,
            Self::Person(_) => RunKind::PersonProperty,
        }
    }

    pub(super) fn team_id(&self) -> TeamId {
        match self {
            Self::Behavioral(prepared) => prepared.run.team_id,
            Self::Person(run) => run.team_id,
        }
    }
}

/// A behavioral run and the static analysis of its conditions, which every chunk of it shares.
///
/// The analysis lives here rather than on the scan because it is a pure function of the pinned
/// payload: a run's answer is the same for all of its chunks, for a chunk retried after a failure,
/// and on whichever replica claims it. Deriving it per chunk instead would re-walk the whole
/// condition catalog up to once per planned day per attempt, on the worker that owes the scan.
pub(super) struct PreparedBehavioral {
    pub(super) run: PinnedRun,
    pub(super) analyses: ConditionAnalyses,
}

/// One refresh pass's result: the claim-eligible runs plus the person runs still needing their
/// planning scan.
pub(super) struct RefreshOutcome {
    pub(super) eligible: HashMap<RunId, PreparedRun>,
    pub(super) planning: Vec<PersonPlanRequest>,
}

/// One discovered run's classification after its boundary/validate/plan pipeline.
enum PrepareOutcome {
    Eligible(PreparedRun),
    WaitingBoundary,
    NoChunks,
    Skipped,
    PlanningNeeded(PersonPlanRequest),
}

pub(super) async fn refresh_runs(
    pool: &PgPool,
    store: &PgChunkStore,
    allowlist: &TeamAllowlist,
    kinds: &[RunKind],
    plan_caps: PlanCaps,
    reported_runs: &mut HashSet<RunId>,
) -> RefreshOutcome {
    let discovered = match discover_runs(pool, allowlist, kinds).await {
        Ok(runs) => runs,
        Err(error) => {
            let run_id = error.run_id();
            handle_run_error(pool, run_id, error).await;
            return RefreshOutcome {
                eligible: HashMap::new(),
                planning: Vec::new(),
            };
        }
    };
    let mut eligible = HashMap::with_capacity(discovered.len());
    let mut planning = Vec::new();
    let mut seen_runs = HashSet::with_capacity(discovered.len());
    let mut waiting_boundary = 0_u64;
    let mut without_chunks: HashMap<RunKind, u64> = HashMap::new();

    for run in discovered {
        seen_runs.insert(run.run_id);
        let kind = run.kind;
        match prepare_run(pool, store, allowlist, plan_caps, reported_runs, run).await {
            PrepareOutcome::Eligible(prepared) => {
                let run_id = match &prepared {
                    PreparedRun::Behavioral(prepared) => prepared.run.run_id,
                    PreparedRun::Person(run) => run.run_id,
                };
                eligible.insert(run_id, prepared);
            }
            PrepareOutcome::WaitingBoundary => waiting_boundary += 1,
            PrepareOutcome::NoChunks => *without_chunks.entry(kind).or_default() += 1,
            PrepareOutcome::Skipped => {}
            PrepareOutcome::PlanningNeeded(request) => {
                *without_chunks.entry(kind).or_default() += 1;
                planning.push(request);
            }
        }
    }

    gauge!(RUNS_WAITING_BOUNDARY).set(waiting_boundary as f64);
    for kind in [RunKind::Behavioral, RunKind::PersonProperty] {
        let count = without_chunks.get(&kind).copied().unwrap_or(0);
        gauge!(RUNS_WITHOUT_CHUNKS, "kind" => kind.as_str()).set(count as f64);
        // Split by kind too: a person run's chunk count tracks its team's person volume, on a
        // scale unrelated to behavioral day-bands, so one series would bury the other's burn-down.
        match store
            .remaining_chunks(&run_ids_of_kind(&eligible, kind))
            .await
        {
            Ok(remaining) => {
                gauge!(RUN_CHUNKS_REMAINING, "kind" => kind.as_str()).set(remaining as f64);
            }
            Err(error) => {
                warn!(error = %error, kind = kind.as_str(), "counting remaining chunks failed");
            }
        }
    }
    reported_runs.retain(|run_id| seen_runs.contains(run_id));
    RefreshOutcome { eligible, planning }
}

async fn prepare_run(
    pool: &PgPool,
    store: &PgChunkStore,
    allowlist: &TeamAllowlist,
    plan_caps: PlanCaps,
    reported_runs: &mut HashSet<RunId>,
    run: DiscoveredRun,
) -> PrepareOutcome {
    counter!(RUNS_DISCOVERED, "status" => run.status.as_str(), "kind" => run.kind.as_str())
        .increment(1);

    let kind = run.kind;
    let Some(boundary) = resolve_boundary(pool, run).await else {
        return PrepareOutcome::Skipped;
    };
    let Resolved::Seedable(boundary) = boundary else {
        return PrepareOutcome::WaitingBoundary;
    };
    match kind {
        RunKind::Behavioral => {
            prepare_behavioral(pool, store, allowlist, plan_caps, reported_runs, boundary).await
        }
        RunKind::PersonProperty => prepare_person(pool, store, reported_runs, boundary).await,
    }
}

enum Resolved {
    Seedable(SeedableRun),
    WaitingBoundary,
}

/// Establish (or re-read) the run's boundary; `None` means the run was skipped.
async fn resolve_boundary(pool: &PgPool, run: DiscoveredRun) -> Option<Resolved> {
    let was_awaiting_boundary = run.status == RunStatus::AwaitingBoundary;
    let discovered_run_id = run.run_id;
    match establish_boundary(pool, run).await {
        Ok(BoundaryOutcome::Established(run)) => {
            counter!(BOUNDARY_ESTABLISHED, "trigger" => run.trigger.as_str()).increment(1);
            Some(Resolved::Seedable(run))
        }
        Ok(BoundaryOutcome::AlreadyEstablished(run)) => {
            if was_awaiting_boundary {
                counter!(BOUNDARY_CAS_LOST).increment(1);
            }
            Some(Resolved::Seedable(run))
        }
        Ok(BoundaryOutcome::NoLongerSeedable { .. }) => None,
        Err(RunError::DisasterRecoveryBoundaryMissing(_)) => Some(Resolved::WaitingBoundary),
        Err(error) => {
            handle_run_error(pool, Some(discovered_run_id), error).await;
            None
        }
    }
}

async fn prepare_behavioral(
    pool: &PgPool,
    store: &PgChunkStore,
    allowlist: &TeamAllowlist,
    plan_caps: PlanCaps,
    reported_runs: &mut HashSet<RunId>,
    boundary: SeedableRun,
) -> PrepareOutcome {
    let run_id = boundary.run_id;
    let phase = boundary.phase;
    let validated = match boundary.load_pinned(pool).await {
        Ok(validated) => validated,
        Err(error) => {
            handle_run_error(pool, Some(run_id), error).await;
            return PrepareOutcome::Skipped;
        }
    };
    let lookback_truncated = lookback_was_truncated(&validated.run, plan_caps);
    let analyses = ConditionAnalyses::build(&validated.run.conditions, &validated.run.filters);
    if reported_runs.insert(run_id) {
        record_pinned_warnings(&validated.warnings);
        record_condition_census(allowlist, run_id, &validated.run, &analyses);
        if lookback_truncated {
            counter!(LOOKBACK_TRUNCATED).increment(1);
        }
    }
    match phase {
        SeedPhase::Seeding => {}
        // Every day was planned while the run was seeding, and its readiness is already stamped.
        SeedPhase::Trailing => return eligible_behavioral(validated.run, analyses),
    }
    if validated
        .warnings
        .iter()
        .any(|warning| matches!(warning, PinnedWarning::ConditionDropped { .. }))
    {
        persist_run_warning(pool, run_id, RunWarningNote::ConditionsDropped).await;
    }
    if lookback_truncated {
        persist_run_warning(pool, run_id, RunWarningNote::LookbackTruncated).await;
    }

    let days = plan_days(
        &validated.run.conditions,
        validated.run.boundary,
        &plan_caps,
    );
    if days.is_empty() {
        // A legitimately zero-chunk run still needs the proof, or it stalls waiting for chunks
        // that will never exist.
        stamp_planning(pool, run_id, RunKind::Behavioral).await;
        return PrepareOutcome::NoChunks;
    }
    let planned = days.into_iter().map(|day| {
        validated
            .run
            .boundary
            .schedule(day, plan_caps.trailing_day_grace)
    });
    match store
        .plan_chunks(validated.run.run_id, planned, plan_caps.bands_per_day)
        .await
    {
        Ok(PlanOutcome::Planned { inserted }) => {
            counter!(CHUNKS_PLANNED, "kind" => RunKind::Behavioral.as_str()).increment(inserted);
            stamp_planning(pool, run_id, RunKind::Behavioral).await;
        }
        Ok(PlanOutcome::RunNotSeeding) => return PrepareOutcome::Skipped,
        Ok(PlanOutcome::AlreadyPlanned) => {
            // Behavioral planning never takes the all-or-nothing gate; reaching this means the
            // store contract changed underneath this caller.
            warn!(
                ?run_id,
                "behavioral planning unexpectedly reported AlreadyPlanned"
            );
            return PrepareOutcome::Skipped;
        }
        Err(error) => {
            warn!(run_id = ?validated.run.run_id, error = %error, "chunk planning failed");
            return PrepareOutcome::Skipped;
        }
    }
    eligible_behavioral(validated.run, analyses)
}

fn eligible_behavioral(run: PinnedRun, analyses: ConditionAnalyses) -> PrepareOutcome {
    PrepareOutcome::Eligible(PreparedRun::Behavioral(Arc::new(PreparedBehavioral {
        run,
        analyses,
    })))
}

/// The person pipeline: validate the pinned payload (an active participation with no surviving
/// hash fails the run inside `handle_run_error`; all-superseded retires as zero-work),
/// then classify by planning state — the stamp or existing chunks make the run claim-eligible;
/// otherwise it needs its planning scan.
async fn prepare_person(
    pool: &PgPool,
    store: &PgChunkStore,
    reported_runs: &mut HashSet<RunId>,
    boundary: SeedableRun,
) -> PrepareOutcome {
    let run_id = boundary.run_id;
    let validated = match boundary.load_person_pinned(pool).await {
        Ok(PersonRunValidation::Seedable(validated)) => validated,
        Ok(PersonRunValidation::Retired { warnings }) => {
            if reported_runs.insert(run_id) {
                record_pinned_warnings(&warnings);
            }
            // Nothing expects coverage, so the zero-chunk proof lets the run finish as zero-work
            // downstream — the behavioral zero-condition retirement, kind-adjusted.
            stamp_planning(pool, run_id, RunKind::PersonProperty).await;
            return PrepareOutcome::NoChunks;
        }
        Err(error) => {
            handle_run_error(pool, Some(run_id), error).await;
            return PrepareOutcome::Skipped;
        }
    };
    if reported_runs.insert(run_id) {
        record_pinned_warnings(&validated.warnings);
        report_person_analysis(run_id, &validated.run);
    }
    if validated
        .warnings
        .iter()
        .any(|warning| matches!(warning, PinnedWarning::ConditionDropped { .. }))
    {
        persist_run_warning(pool, run_id, RunWarningNote::ConditionsDropped).await;
    }
    let stamped = match read_planning_stamp(pool, run_id, RunKind::PersonProperty).await {
        Ok(stamp) => stamp.is_some(),
        Err(error) => {
            warn!(?run_id, error = %error, "reading the person planning stamp failed");
            return PrepareOutcome::Skipped;
        }
    };
    let run = Arc::new(validated.run);
    if stamped {
        return PrepareOutcome::Eligible(PreparedRun::Person(run));
    }
    match store.chunk_progress(run_id).await {
        Ok(progress) if progress.total() > 0 => {
            // A prior attempt landed the chunks and left the stamp to this pass.
            stamp_planning(pool, run_id, RunKind::PersonProperty).await;
            PrepareOutcome::Eligible(PreparedRun::Person(run))
        }
        Ok(_) => PrepareOutcome::PlanningNeeded(PersonPlanRequest { run }),
        Err(error) => {
            warn!(?run_id, error = %error, "reading person chunk progress failed");
            PrepareOutcome::Skipped
        }
    }
}

/// One line per run naming what its conditions need from a row and whether ClickHouse can drop
/// key-less persons for it.
///
/// `prunable_keys` is what the relevant-only policy would filter on; whether it actually did is the
/// per-chunk `seeder_person_scan_filter_total{outcome}` reading. Together they separate "the run
/// cannot be filtered" from "this deployment is healing".
fn report_person_analysis(run_id: RunId, run: &PinnedPersonRun) {
    let census = run.analysis_census();
    let prunable_keys = run.scan_key_filter(PersonEmissionPolicy::RelevantToSomeCohort);
    if run.composable_cohorts() == Some(0) {
        // Every participation is structurally excluded, so the consumer composes none of them and
        // no seed this run emits registers membership for a cohort of its own. It will scan the
        // team and confirm every chunk, which reads exactly like a successful backfill.
        warn!(
            ?run_id,
            "person run composes no cohort; no participation of its own registers membership",
        );
    }
    info!(
        ?run_id,
        conditions = run.conditions.len(),
        key_decidable = census.key_decidable,
        always_evaluate = %census.render_always_evaluate(),
        composable_cohorts = ?run.composable_cohorts(),
        prunable_key_count = ?prunable_keys.as_ref().map(ProjectedKeys::count),
        "person run analyzed",
    );
    // The key names are customer-defined and uncapped, so they follow the same rule as the
    // behavioral census's read sets below.
    debug!(
        ?run_id,
        prunable_keys = ?prunable_keys.as_ref().map(|keys| keys.iter().collect::<Vec<_>>()),
        "person run prunable keys",
    );
}

pub(super) fn run_ids_of_kind(
    eligible_runs: &HashMap<RunId, PreparedRun>,
    kind: RunKind,
) -> Vec<RunId> {
    eligible_runs
        .iter()
        .filter(|(_, prepared)| prepared.kind() == kind)
        .map(|(run_id, _)| *run_id)
        .collect()
}

/// Stamp the planning proof once planning has run. A failure is logged but never changes the prepare
/// outcome — the run keeps seeding and the next pass re-stamps.
async fn stamp_planning(pool: &PgPool, run_id: RunId, kind: RunKind) {
    match mark_chunks_planned(pool, run_id, kind).await {
        Ok(PlanningStampOutcome::Stamped) => {
            counter!(RUNS_PLANNING_STAMPED, "kind" => kind.as_str()).increment(1);
        }
        Ok(PlanningStampOutcome::Skipped) => {}
        Err(error) => warn!(run_id = ?run_id, error = %error, "stamping the planning proof failed"),
    }
}

async fn handle_run_error(pool: &PgPool, run_id: Option<RunId>, error: RunError) {
    let disposition = run_error_disposition(run_id, &error);
    if let RunErrorDisposition::Fail { run_id, reason } = disposition {
        counter!(RUN_VALIDATION_FAILURES, "reason" => reason).increment(1);
        let detail = RenderedError::render(&error);
        if let Err(failure) = fail_run(pool, run_id, &detail).await {
            warn!(run_id = ?run_id, error = %failure, "failing invalid run did not apply");
        }
        return;
    }
    warn!(error = %error, "transient run preparation failed");
}

async fn persist_run_warning(pool: &PgPool, run_id: RunId, note: RunWarningNote) {
    if let Err(error) = record_run_warning(pool, run_id, note).await {
        warn!(run_id = ?run_id, error = %error, "persisting run warning failed");
    }
}

/// Publish what a static read of the run's condition bytecode found: which event names a projection
/// narrows, and what blocks the rest.
///
/// Reports once per run per stretch, behind the `reported_runs` gate, while the analysis it reads is
/// built for every eligible run on every poll tick — the scan needs it, so it cannot sit behind a
/// report gate. That cost is bounded per run rather than per condition, which matters because
/// nothing caps behavioral conditions on a run and this runs inline on the task that owes the
/// liveness heartbeat: `ConditionAnalyses::build` gives the run's conditions one shared budget. The
/// same tick already parses every discovered run's whole pinned payload during validation, which is
/// more work than analyzing that payload once.
fn record_condition_census(
    allowlist: &TeamAllowlist,
    run_id: RunId,
    run: &PinnedRun,
    analyses: &ConditionAnalyses,
) {
    let census = analyses.census(&run.conditions);
    if census.total() == 0 {
        return;
    }
    let team = team_label(allowlist, run.team_id);
    for class in ConditionClass::ALL {
        let count = census.count(class);
        if count > 0 {
            counter!(
                CONDITIONS_CLASSIFIED,
                "class" => class.as_str(),
                "team_id" => team.clone(),
            )
            .increment(count);
        }
    }
    for (label, count) in &census.unanalyzable_reasons {
        counter!(
            CONDITIONS_UNANALYZABLE,
            "reason" => label.reason,
            "op" => label.op.clone().unwrap_or_else(|| Arc::from("none")),
            "team_id" => team.clone(),
        )
        .increment(*count);
    }
    info!(
        ?run_id,
        team_id = run.team_id.0,
        conditions = census.total(),
        event_only = census.count(ConditionClass::EventOnly),
        projectable = census.count(ConditionClass::Projectable),
        full_columns = census.count(ConditionClass::FullColumns),
        unanalyzable = census.count(ConditionClass::Unanalyzable),
        property_projectable_fraction = census.property_projectable_fraction(),
        eligible_events = census.projection_eligible_event_names().len(),
        blocked_events = %census.render_blocked_events(),
        row_filtered_conditions = analyses.row_filtered_conditions(),
        "condition bytecode analysis census",
    );
    // The read sets and the uncapped blocked list carry customer-defined event names and property
    // keys, and run to several kilobytes on a large catalog. Values are never included, but the
    // keys alone are enough to keep this off the default level. Named apart from the line above's
    // `blocked_events` because that one is truncated and this one is not, and a query that mixed
    // them would read a partial list as a whole one.
    debug!(
        ?run_id,
        reads = %census.render_reads(),
        all_blocked_events = %census.render_all_blocked_events(),
        "condition bytecode analysis reads and blockers",
    );
}

fn record_pinned_warnings(warnings: &[PinnedWarning]) {
    for warning in warnings {
        match warning {
            PinnedWarning::TimezoneFallback { .. } => counter!(TZ_FALLBACK).increment(1),
            PinnedWarning::ConditionDropped { reason, .. } => {
                counter!(CONDITIONS_DROPPED, "reason" => reason.as_str()).increment(1);
            }
            PinnedWarning::ConditionSuperseded { .. } => {
                counter!(CONDITIONS_DROPPED, "reason" => "superseded_participation").increment(1);
            }
            PinnedWarning::WindowDaysMismatch { .. } => {
                counter!(WINDOW_DAYS_MISMATCH).increment(1);
            }
        }
    }
}

fn lookback_was_truncated(run: &PinnedRun, caps: PlanCaps) -> bool {
    let capped_start = window_start_for_now(run.boundary.day(), caps.max_lookback_days);
    run.conditions
        .iter()
        .any(|condition| match condition.lookback {
            Lookback::SlidingDays(days) => days > caps.max_lookback_days,
            Lookback::FixedRange { from_day: None, .. } => true,
            Lookback::FixedRange {
                from_day: Some(from),
                ..
            } => from < capped_start,
            Lookback::SubDay => false,
        })
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RunErrorDisposition {
    Retry,
    Fail { run_id: RunId, reason: &'static str },
}

fn run_error_disposition(run_id: Option<RunId>, error: &RunError) -> RunErrorDisposition {
    match error {
        RunError::Pg(_) => RunErrorDisposition::Retry,
        // Never a retry: the run re-derives the same refusal on every poll tick.
        RunError::Pinned(PinnedError::UncoveredParticipations(_)) => {
            run_id.map_or(RunErrorDisposition::Retry, |run_id| {
                RunErrorDisposition::Fail {
                    run_id,
                    reason: "uncovered_participation",
                }
            })
        }
        RunError::Pinned(_) => run_id.map_or(RunErrorDisposition::Retry, |run_id| {
            RunErrorDisposition::Fail {
                run_id,
                reason: "pinned_validation",
            }
        }),
        RunError::CrossTeamParticipation { run_id, .. } => RunErrorDisposition::Fail {
            run_id: *run_id,
            reason: "cross_team_participation",
        },
        RunError::SeedingBoundaryMissing(run_id) => RunErrorDisposition::Fail {
            run_id: *run_id,
            reason: "missing_boundary",
        },
        RunError::DisasterRecoveryBoundaryMissing(_) => RunErrorDisposition::Retry,
        RunError::InvalidKind { run_id, .. }
        | RunError::InvalidTrigger { run_id, .. }
        | RunError::InvalidScope { run_id, .. }
        | RunError::InvalidStatus { run_id, .. } => RunErrorDisposition::Fail {
            run_id: *run_id,
            reason: "invalid_run_row",
        },
        RunError::UnknownStatus(_)
        | RunError::UnknownScope(_)
        | RunError::NotFound(_)
        | RunError::NotActive(_) => RunErrorDisposition::Retry,
    }
}

#[cfg(test)]
mod tests {
    use cohort_core::filters::CohortId;
    use cohort_core::leaf_state::select::UnsupportedVariant;
    use uuid::Uuid;

    use crate::domain::{
        ConditionHash, PinnedDropReason, UncoveredCohort, UncoveredParticipations, UncoveredReason,
    };

    use super::*;

    #[test]
    fn an_uncovered_participation_fails_its_run_under_its_own_reason() {
        let run_id = RunId(Uuid::nil());
        let uncovered = RunError::Pinned(PinnedError::UncoveredParticipations(
            UncoveredParticipations(vec![UncoveredCohort {
                cohort_id: CohortId(574801),
                reason: UncoveredReason::NoSurvivingCondition,
                catalog_class: Some("excluded_has_dropped_leaf"),
                dropped: vec![(
                    ConditionHash::parse("c8236865303eb463").unwrap(),
                    PinnedDropReason::UnsupportedStateVariant(UnsupportedVariant::MissingWindow),
                )],
            }]),
        ));
        assert_eq!(
            run_error_disposition(Some(run_id), &uncovered),
            RunErrorDisposition::Fail {
                run_id,
                reason: "uncovered_participation",
            }
        );

        let malformed = RunError::Pinned(PinnedError::SchemaVersion(2));
        assert_eq!(
            run_error_disposition(Some(run_id), &malformed),
            RunErrorDisposition::Fail {
                run_id,
                reason: "pinned_validation",
            }
        );
    }
}