Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

54.1 KB
Raw Blame
"""Rig CLI — the single entry point for tox, pre-commit, and CI.

Subcommands:
  run                 Execute selected groups (with dep-expansion).
  list-groups         Print known groups, tiers, and dep edges.
  list-critical-paths Print critical-path coverage status (best-effort).
  expand              Print the topo-sorted set of groups that ``run`` would execute.
  validate            Validate manifests; non-zero on schema errors.
  platform            ``up | down | status`` the e2e platform stack.
  report              ``combine`` — re-aggregate reports/ after the fact.
"""

from __future__ import annotations

import argparse
import os
import shutil
import subprocess
import sys
import time
import uuid
from functools import lru_cache
from pathlib import Path
from urllib.parse import urlsplit
from xml.sax import saxutils

from tests.rig import critical_paths as cp
from tests.rig.coverage import combine_and_report, coverage_env
from tests.rig.groups import (
    REPO_ROOT,
    TIERS,
    GroupDefinition,
    MigrateSpec,
    load_groups,
)
from tests.rig.reporting import (
    STATUS_PASS,
    GroupResult,
    parse_junit,
    passed_critical_path_ids,
    write_summary,
)
from tests.rig.runtime import (
    DEFAULT_LLM_MOCK_RESPONSE,
    LLM_MOCK_RESPONSE_ENV,
    PlatformEndpoints,
    PlatformRuntime,
    TestcontainersRuntime,
    pick_runtime,
)
from tests.rig.selection import resolve

# Pytest exit codes that the rig treats as non-failure for aggregation:
#   0 — all tests passed
#   5 — no tests collected (optional placeholders, empty hurl group, etc.)
#       NOTE: exit-5 is non-failing here for *optional* groups only. A required
#       group that collects nothing (exit 5) is caught by the explicit guard in
#       cmd_run ("Empty collection (exit 5) is a failure…") and fails loudly.
_NON_FAILING_PYTEST_EXIT_CODES = (0, 5)

# Injected into every group's pytest run (PYTHONPATH + -p) so
# @pytest.mark.critical_path marker args land in junit properties.
_PYTEST_PLUGIN_DIR = REPO_ROOT / "tests" / "rig" / "pytest_plugin"

# saxutils.escape leaves quotes alone, which is wrong inside an attribute.
_XML_ATTR_ESCAPES = {'"': """}

# Runs outside any group's timeout budget, so a wedged connection would
# otherwise hang the job to the CI ceiling.
_MIGRATE_TIMEOUT_SECONDS = 300


@lru_cache(maxsize=1)
def _rig_session_id() -> str:
    """Stable per-invocation sentinel, computed once.

    Stamped into ``UNSTRACT_RIG_SESSION_ID`` for every group's pytest env so
    e2e tests can prove the rig ran. URL ownership is intentionally cooperative
    — the rig sets ``UNSTRACT_*_URL`` via ``setdefault``, so a developer's
    pre-set value wins (see tests/README.md). The session id is the rig's
    signature, not a claim that the rig owns the URLs.
    """
    return uuid.uuid4().hex


def _subprocess_env() -> dict[str, str]:
    """Base environment for every group's ``uv``/pytest subprocess.

    Drops ``VIRTUAL_ENV`` so ``uv run`` does clean per-group project discovery
    (each group runs against its own workdir's ``.venv``). When the rig runs
    under tox, tox exports ``VIRTUAL_ENV=.tox/<env>``, which doesn't match any
    group's project and makes ``uv`` emit a "does not match the project
    environment path" warning before ignoring it anyway. Stripping it removes
    the noise and the ambiguity. ``UV_PROJECT_ENVIRONMENT`` is dropped for the
    same reason.
    """
    env = os.environ.copy()
    env.pop("VIRTUAL_ENV", None)
    env.pop("UV_PROJECT_ENVIRONMENT", None)
    return env


def main(argv: list[str] | None = None) -> int:
    parser = _build_parser()
    args = parser.parse_args(argv)
    return args.func(args)


def _default_workers() -> str:
    """Resolve the xdist worker count ourselves.

    xdist's own `auto` prefers *physical* cores when psutil happens to be
    importable, which collapses to a single worker on hyperthreaded CI runners
    — and only for the groups that ship psutil. Beyond ~8 the suites contend on
    the test database more than they parallelise, so cap it.
    """
    try:
        cpus = len(os.sched_getaffinity(0))  # honours CPU pinning; Linux only
    except AttributeError:
        cpus = os.cpu_count() or 1
    return str(min(cpus, 8))


def _build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(prog="tests.rig", description="Unstract unified test rig")
    sub = p.add_subparsers(dest="command", required=True)

    pr = sub.add_parser("run", help="Run selected groups")
    pr.add_argument("groups", nargs="*", help="Group names, or 'all'.")
    pr.add_argument("--from-file", type=Path, help="File with one group name per line.")
    pr.add_argument("--tier", choices=TIERS)
    pr.add_argument("--marker", help="Pytest -m marker expression to forward.")
    pr.add_argument(
        "--paths", help="Comma-separated pytest paths/nodeids (overrides group paths)."
    )
    pr.add_argument("--runtime", choices=["compose", "testcontainers", "local"])
    pr.add_argument("--coverage", dest="coverage", action="store_true", default=True)
    pr.add_argument("--no-coverage", dest="coverage", action="store_false")
    pr.add_argument("--parallel", dest="parallel", action="store_true", default=True)
    pr.add_argument("--no-parallel", dest="parallel", action="store_false")
    pr.add_argument(
        "--workers",
        default=_default_workers(),
        help="pytest-xdist worker count (default: usable CPUs, capped).",
    )
    pr.add_argument("--timeout", type=int, help="Per-group timeout override in seconds.")
    pr.add_argument("--reports-dir", type=Path, default=REPO_ROOT / "reports")
    pr.add_argument(
        "--fail-on-critical-gap",
        action="store_true",
        help="Treat uncovered critical paths as a build failure.",
    )
    pr.add_argument(
        "--changed-only",
        action="store_true",
        help="Auto-select groups overlapping `git diff origin/main...HEAD`.",
    )
    pr.add_argument("--changed-base", default="origin/main")
    pr.add_argument(
        "--baseline",
        type=Path,
        default=REPO_ROOT / "reports" / "previous-summary.json",
    )
    pr.add_argument(
        "--update-baseline",
        action="store_true",
        help=(
            "On green main builds, merge this build's covered paths into the "
            "baseline. Merging (not overwriting) preserves coverage recorded by "
            "earlier tier invocations in the same workflow."
        ),
    )
    pr.add_argument("--dry-run", action="store_true", help="Plan only; do not execute.")
    pr.set_defaults(func=cmd_run)

    pl = sub.add_parser("list-groups", help="List all defined groups.")
    pl.set_defaults(func=cmd_list_groups)

    pc = sub.add_parser("list-critical-paths", help="Print critical-path status table.")
    pc.add_argument(
        "--baseline",
        type=Path,
        default=REPO_ROOT / "reports" / "previous-summary.json",
    )
    pc.set_defaults(func=cmd_list_critical)

    pe = sub.add_parser("expand", help="Show what `run` would execute, in topo order.")
    pe.add_argument("groups", nargs="*")
    pe.add_argument("--from-file", type=Path)
    pe.add_argument("--tier", choices=TIERS)
    pe.add_argument("--changed-only", action="store_true")
    pe.add_argument("--changed-base", default="origin/main")
    pe.set_defaults(func=cmd_expand)

    pv = sub.add_parser("validate", help="Validate manifests.")
    pv.set_defaults(func=cmd_validate)

    pp = sub.add_parser("platform", help="Manage the e2e platform stack.")
    pp.add_argument("action", choices=["up", "down", "status"])
    pp.add_argument("--runtime", choices=["compose", "testcontainers", "local"])
    pp.set_defaults(func=cmd_platform)

    pre = sub.add_parser("report", help="Re-aggregate existing reports/.")
    pre.add_argument("action", choices=["combine"])
    pre.add_argument("--reports-dir", type=Path, default=REPO_ROOT / "reports")
    pre.add_argument(
        "--baseline",
        type=Path,
        default=None,
        help="Baseline path (default: <reports-dir>/previous-summary.json).",
    )
    pre.add_argument(
        "--update-baseline",
        action="store_true",
        help=(
            "Merge covered paths from all tiers' reports into the baseline in "
            "one write (parallel per-tier writes would race). Refused (exit 1) "
            "if any non-optional group is red or the baseline is corrupt."
        ),
    )
    pre.set_defaults(func=cmd_report)

    return p


# ── subcommands ───────────────────────────────────────────────────────────────


def cmd_list_groups(_args: argparse.Namespace) -> int:
    manifest = load_groups()
    for name in manifest.names():
        g = manifest.get(name)
        deps = ", ".join(g.depends_on) or "—"
        flags: list[str] = []
        if g.critical:
            flags.append("critical")
        if g.requires_platform:
            flags.append("platform")
        if g.optional:
            flags.append("optional")
        if g.requires_services:
            flags.append("svc:" + "+".join(g.requires_services))
        print(
            f"  {name:<32} tier={g.tier:<11} runner={g.runner:<7} "
            f"deps=[{deps}]  {' '.join(flags)}"
        )
    return 0


def cmd_list_critical(args: argparse.Namespace) -> int:
    manifest = load_groups()
    registry = cp.load_critical_paths()
    errors = cp.validate_registry_against_manifest(registry, manifest)
    for err in errors:
        print(f"ERROR: {err}", file=sys.stderr)
    try:
        baseline = cp.load_baseline(args.baseline)
    except cp.BaselineCorruptError as exc:
        print(f"[rig] {exc}", file=sys.stderr)
        baseline = None
    statuses = cp.evaluate(registry, groups_run_green=set(), baseline=baseline)
    icons = {"covered": "✅", "gap": "⚠️", "regression": "❌"}
    for s in statuses:
        cov = ", ".join(s.path.covered_by) or "—"
        print(f"  {icons[s.state]} {s.path.id:<28} declared coverage: {cov}")
    return 1 if errors else 0


def cmd_expand(args: argparse.Namespace) -> int:
    manifest = load_groups()
    ordered = resolve(
        manifest,
        positional=args.groups or [],
        from_file=args.from_file,
        tier=args.tier,
        changed_only=args.changed_only,
        changed_base=args.changed_base,
    )
    if not ordered:
        print("(no groups selected)", file=sys.stderr)
        return 1
    for name in ordered:
        g = manifest.get(name)
        print(f"  {name}  (tier={g.tier}, runner={g.runner})")
    return 0


def cmd_validate(_args: argparse.Namespace) -> int:
    """Validate both manifests. Schema/path/cycle errors come from
    ``load_groups`` (raised); cross-manifest errors come from
    ``validate_registry_against_manifest``.
    """
    try:
        manifest = load_groups()
    except (ValueError, FileNotFoundError) as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        return 1
    try:
        registry = cp.load_critical_paths()
    except (ValueError, FileNotFoundError) as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        return 1
    errors = cp.validate_registry_against_manifest(registry, manifest)
    # Config-isolation is filesystem-dependent, so it lives here rather than in
    # the pure loader — but running it at validate time catches a misconfigured
    # group in review instead of mid-run, after infra/dep bring-up.
    for name in manifest.names():
        group = manifest.get(name)
        # Optional groups may be placeholders whose paths don't exist yet.
        if group.optional:
            continue
        isolation_error = _config_isolation_error(group, group.absolute_workdir())
        if isolation_error is not None:
            errors.append(isolation_error)
    for err in errors:
        print(f"ERROR: {err}", file=sys.stderr)
    if errors:
        return 1
    print(f"OK — {len(manifest.names())} groups, {len(registry.paths)} critical paths")
    return 0


def cmd_platform(args: argparse.Namespace) -> int:
    runtime = pick_runtime(args.runtime)
    if args.action == "up":
        try:
            endpoints = runtime.up()
        except Exception:
            # If up() raised mid-way (e.g. one testcontainer started, the next
            # failed), down() cleans up the partial stack.
            runtime.down()
            raise
        print(f"Platform up via runtime={runtime.name}:")
        print(f"  backend         : {endpoints.backend_url}")
        print(f"  prompt-service  : {endpoints.prompt_service_url}")
        print(f"  platform-service: {endpoints.platform_service_url}")
        print(f"  runner          : {endpoints.runner_url}")
        return 0
    if args.action == "down":
        runtime.down()
        return 0
    if args.action == "status":
        print(
            f"runtime={runtime.name} (status check is best-effort; "
            f"see `docker compose ps` for compose)"
        )
        return 0
    return 2


def cmd_report(args: argparse.Namespace) -> int:
    reports_dir: Path = args.reports_dir
    combine_and_report(reports_dir)
    manifest = load_groups()
    registry = cp.load_critical_paths()
    group_results: list[GroupResult] = []
    for name in manifest.names():
        tier = manifest.get(name).tier
        result = parse_junit(name, tier, reports_dir)
        if result is not None:
            group_results.append(result)
    green = _coverage_attesting_groups(group_results)
    proven, unknown_marker_ids = _marker_proven_paths(
        registry, group_results, reports_dir
    )
    baseline_path = args.baseline or reports_dir / "previous-summary.json"
    baseline_corrupt = False
    try:
        baseline = cp.load_baseline(baseline_path)
    except cp.BaselineCorruptError as exc:
        print(f"[rig] {exc}", file=sys.stderr)
        baseline = None
        baseline_corrupt = True
    # A skipped tier emits no junit, so scoping to the groups that reported keeps
    # its paths as gaps rather than regressions — nothing regressed, it never ran.
    # A group that ran and went red still reports, so real regressions survive.
    # `optional` groups are excluded: they are documented as non-blocking, and
    # leaving them in scope would let a red one gate the build through a
    # regression instead.
    scope_groups = [r.name for r in group_results if not manifest.get(r.name).optional]
    statuses = cp.evaluate(
        registry,
        groups_run_green=green,
        baseline=baseline,
        scope_groups=scope_groups,
        marker_proven=proven,
    )
    write_summary(
        reports_dir=reports_dir,
        group_results=group_results,
        critical_statuses=statuses,
        baseline_corrupt=baseline_corrupt,
    )
    print(f"Wrote {reports_dir / 'summary.md'}")
    if unknown_marker_ids:
        print(
            f"[rig] ❌ @pytest.mark.critical_path references unknown path "
            f"id(s): {', '.join(unknown_marker_ids)} "
            f"(not in tests/critical_paths.yaml)",
            file=sys.stderr,
        )
    # cmd_run gates on the regressions it can see, but only within its own tier.
    # This is the only cross-tier evaluation, so it is the only place a regression
    # spanning tiers can be gated on.
    # A covering group that ran green without attesting means its marked test
    # was skipped or unmarked; a group that went red means the test failed. The
    # remedies differ, so the two are reported apart rather than as one count.
    unproven: list[cp.CriticalPathStatus] = []
    uncovered: list[cp.CriticalPathStatus] = []
    for s in statuses:
        if s.state != "regression":
            continue
        target = unproven if any(g in green for g in s.path.covered_by) else uncovered
        target.append(s)
    regressions = unproven + uncovered
    if uncovered:
        ids = ", ".join(s.path.id for s in uncovered)
        print(
            f"\n[rig] ❌ {len(uncovered)} critical-path regression(s) — no covering "
            f"group ran green: {ids}",
            file=sys.stderr,
        )
    if unproven:
        ids = ", ".join(s.path.id for s in unproven)
        print(
            f"\n[rig] ❌ {len(unproven)} critical-path regression(s) — covering group "
            f"ran green but no passing @pytest.mark.critical_path test attested "
            f"them (skipped or unmarked?): {ids}",
            file=sys.stderr,
        )
    if regressions:
        print(
            "[rig] to accept a deliberate removal, drop or re-point the path in "
            "tests/critical_paths.yaml in the same PR — the baseline is keyed off "
            "that registry.",
            file=sys.stderr,
        )
    if args.update_baseline:
        red = [
            r.name
            for r in group_results
            if r.status not in ("pass", "empty") and not manifest.get(r.name).optional
        ]
        if baseline_corrupt or red:
            reason = (
                "corrupt baseline"
                if baseline_corrupt
                else f"red groups: {', '.join(red)}"
            )
            print(f"[rig] ❌ baseline update refused ({reason})", file=sys.stderr)
            return 1
        cp.merge_into_baseline(statuses, baseline_path)
        print(f"[rig] merged into baseline: {baseline_path}")
    # A corrupt baseline makes "regression" unreachable, so the gate above would
    # pass vacuously; it has to fail on its own.
    return 1 if unknown_marker_ids or regressions or baseline_corrupt else 0


def cmd_run(args: argparse.Namespace) -> int:
    manifest = load_groups()
    registry = cp.load_critical_paths()
    errs = cp.validate_registry_against_manifest(registry, manifest)
    for err in errs:
        print(f"ERROR: {err}", file=sys.stderr)
    if errs:
        return 2

    ordered = resolve(
        manifest,
        positional=args.groups or [],
        from_file=args.from_file,
        tier=args.tier,
        changed_only=args.changed_only,
        changed_base=args.changed_base,
    )
    if not ordered:
        print(
            "ERROR: no groups selected. Pass group names, `all`, --tier, or --from-file.",
            file=sys.stderr,
        )
        return 2

    runnable = [n for n in ordered if not _is_missing_placeholder(manifest.get(n))]
    skipped = [n for n in ordered if n not in runnable]
    for n in skipped:
        print(f"SKIP {n} (optional + workdir/paths absent)")
    # Scope of this invocation = every group we will actually run AFTER
    # dep-expansion (includes dependencies the user didn't ask for directly).
    # Skipped optional placeholders are excluded: their critical paths were
    # never going to be exercised here, so they must classify as gap, not
    # regression. evaluate() uses this to distinguish "this path's group ran
    # red" (regression) from "this path belongs to a tier we weren't running
    # this time" (gap).
    scope_groups = set(runnable)

    reports_dir: Path = args.reports_dir
    reports_dir.mkdir(parents=True, exist_ok=True)

    needs_platform = any(manifest.get(n).requires_platform for n in runnable)
    # Every runtime (not just compose) needs the mock value present for the
    # execute-path e2e, and an exported empty string counts as unset — else the
    # workers skip injection and the tests skip green with zero coverage.
    if needs_platform and not os.environ.get(LLM_MOCK_RESPONSE_ENV):
        os.environ[LLM_MOCK_RESPONSE_ENV] = DEFAULT_LLM_MOCK_RESPONSE
    # A group can declare `requires_services` (stateful infra like Postgres/
    # Redis) without needing the whole platform — provision just that infra via
    # testcontainers instead of standing up every compose service. Platform wins
    # when both are set.
    needs_services = any(manifest.get(n).requires_services for n in runnable)
    runtime: PlatformRuntime | None = None
    endpoints: PlatformEndpoints | None = None
    group_results: list[GroupResult] = []
    overall_exit = 0

    try:
        if needs_platform and not args.dry_run:
            runtime = pick_runtime(args.runtime)
            print(f"[rig] bringing platform up via runtime={runtime.name}")
            # `up()` is inside the try so a failure here still triggers `down()`
            # in the finally, cleaning up any partial stack.
            endpoints = runtime.up()
        elif needs_services and not args.dry_run:
            # Infra-only: testcontainers Postgres/Redis/etc., no platform.
            runtime = TestcontainersRuntime()
            print(
                f"[rig] bringing infra up via runtime={runtime.name} (requires_services)"
            )
            endpoints = runtime.up()

        # Guarded on `postgres_url` so it only fires for a rig-owned container
        # (under compose the platform migrates its own database on startup), and
        # on a runnable group actually needing Postgres, since `up()` provisions
        # the full infra set even for a run that only wants another service.
        needs_postgres = any(
            "postgres" in manifest.get(n).requires_services for n in runnable
        )
        if (
            endpoints is not None
            and manifest.postgres_migrate is not None
            and needs_postgres
        ):
            postgres_url = endpoints.infra.postgres_url
            if postgres_url:
                migrate_exit = _run_migrations(
                    manifest.postgres_migrate, postgres_url=postgres_url
                )
                if migrate_exit != 0:
                    # Every DB-backed test would skip on a missing table, which
                    # reads as "nothing to run" rather than a broken database.
                    raise RuntimeError(
                        f"migrations failed against the provisioned database "
                        f"(exit {migrate_exit}); aborting before any group runs"
                    )

        # Groups run in topo order, so a dependency's result is always known by
        # the time its dependents come up. A red dep means the precondition it
        # gates (e.g. e2e-smoke: is the platform actually up?) does not hold, so
        # running its dependents only yields noise against a half-up stack.
        #
        # `optional` cascades like any other dep: it governs whether a failure
        # gates CI, not whether the stack can be trusted. The skip itself never
        # sets overall_exit — the failing dep already did if it was non-optional,
        # and a blocked group attests no coverage, so --fail-on-critical-gap
        # still catches a critical path that silently stopped being covered.
        # Groups something else depends on: an empty run (exit 5) of one of
        # these leaves its precondition unverified, so it must block dependents
        # rather than reading as a clean pass.
        depended_on: set[str] = set()
        for n in runnable:
            depended_on |= manifest.transitive_deps(n)
        failed_groups: set[str] = set()
        for name in runnable:
            group = manifest.get(name)
            blocked_by = tuple(sorted(manifest.transitive_deps(name) & failed_groups))
            if blocked_by:
                print(
                    f"\n[rig] SKIP {name} "
                    f"(dependency failed or was blocked: {', '.join(blocked_by)})"
                )
                group_results.append(_blocked_result(group, blocked_by))
                failed_groups.add(name)
                if not args.dry_run:
                    _persist_blocked_group(reports_dir, group, blocked_by)
                continue
            print(
                f"\n[rig] running group: {name} "
                f"(tier={group.tier}, runner={group.runner})"
            )
            if args.dry_run:
                continue
            result, exit_code = _execute_group(
                group,
                reports_dir=reports_dir,
                marker=args.marker,
                paths_override=args.paths,
                coverage=args.coverage,
                parallel=args.parallel and group.parallel,
                workers=args.workers,
                timeout=args.timeout or group.timeout_seconds,
                endpoints=endpoints,
            )
            if result is not None:
                group_results.append(result)
            # Tracked regardless of `optional` — see the cascade note above.
            # An empty run (exit 5) blocks dependents only when something depends
            # on this group: a gate that collected nothing verified nothing, but
            # a leaf placeholder collecting nothing gates no one.
            if (
                exit_code not in _NON_FAILING_PYTEST_EXIT_CODES
                or (result is not None and (result.errors or result.failed))
                or (exit_code == 5 and name in depended_on)
            ):
                failed_groups.add(name)
            # `optional: true` groups run and surface their result in the
            # summary, but never gate the overall exit. This honors the
            # developer intent for groups that need infra we don't provision in
            # CI (live-DB connector tests) or that are pluggable/cloud-only:
            # red shows in the report, merge isn't blocked. Both exit-folds
            # below are gated on `not group.optional` so the skip is consistent
            # whether the failure came via exit code or junit attestation.
            #
            # Always fold the exit code into overall_exit, even when junit.xml
            # was never written (segfault/OOM/missing binary). Otherwise the
            # rig silently returns 0 for catastrophic group failures.
            if (
                exit_code not in _NON_FAILING_PYTEST_EXIT_CODES
                and not group.optional
                and overall_exit == 0
            ):
                overall_exit = exit_code
            # A non-optional pytest group that collected zero tests (exit 5) is
            # almost always a broken marker/path (e.g. the `unit-workers`
            # regression where `-m "unit"` matched zero tests), not intent — fail
            # rather than fold in green. Skip hurl (its exit 5 means "no files", a
            # placeholder) and dev runs with a marker/paths override that may
            # legitimately match nothing.
            if (
                exit_code == 5
                and not group.optional
                and group.runner == "pytest"
                and not args.marker
                and not args.paths
                and overall_exit == 0
            ):
                overall_exit = 1
            # Belt-and-braces: if the junit attests to errors/failures the exit
            # code didn't (truncated junit → errors=1 with exit 0), the report
            # shows ❌ but exit would otherwise stay 0. Keep them in sync.
            if (
                result is not None
                and (result.errors or result.failed)
                and not group.optional
                and overall_exit == 0
            ):
                overall_exit = 1
    finally:
        if runtime is not None and not args.dry_run:
            print(f"[rig] tearing platform down (runtime={runtime.name})")
            # Don't let a teardown failure mask the in-flight exception.
            # Python re-raises whatever exception we caught here if down()
            # raises during a `finally`, hiding the real cause upstream.
            try:
                runtime.down()
            except Exception as exc:
                print(
                    f"[rig] teardown failed (runtime={runtime.name}): {exc}",
                    file=sys.stderr,
                )

    if args.coverage and not args.dry_run:
        combine_and_report(reports_dir)

    green = _coverage_attesting_groups(group_results)
    proven, unknown_marker_ids = _marker_proven_paths(
        registry, group_results, reports_dir
    )
    if unknown_marker_ids:
        print(
            f"[rig] ❌ @pytest.mark.critical_path references unknown path "
            f"id(s): {', '.join(unknown_marker_ids)} "
            f"(not in tests/critical_paths.yaml)",
            file=sys.stderr,
        )
        if overall_exit == 0:
            overall_exit = 1
    # A corrupt baseline can't be silently treated as empty (that would turn
    # the next build into a regression festival once a one-tier baseline gets
    # written back). But we must still write the per-group summary so the
    # developer can see what passed/failed — otherwise the reporting path
    # silently fails on top of the baseline error.
    baseline_corrupt = False
    try:
        baseline = cp.load_baseline(args.baseline)
    except cp.BaselineCorruptError as exc:
        print(f"[rig] ❌ {exc}", file=sys.stderr)
        baseline = None
        baseline_corrupt = True

    statuses = cp.evaluate(
        registry,
        groups_run_green=green,
        baseline=baseline,
        scope_groups=scope_groups,
        marker_proven=proven,
    )
    write_summary(
        reports_dir=reports_dir,
        group_results=group_results,
        critical_statuses=statuses,
        baseline_corrupt=baseline_corrupt,
    )

    # Surface baseline corruption regardless of whether the build was
    # otherwise green or red. A red-then-fixed cycle without this flip would
    # silently swallow the corrupt cache and disable regression detection on
    # the next N builds.
    if baseline_corrupt and overall_exit == 0:
        overall_exit = 1

    # In a dry run no groups executed, so every covered path looks like a
    # gap/regression. A dry run is plan-only: report but never fail (and never
    # write a baseline) on the back of results that didn't happen.
    if args.dry_run:
        return overall_exit

    regressions = [s for s in statuses if s.state == "regression"]
    if regressions:
        print(
            f"\n[rig] ❌ {len(regressions)} critical-path regression(s) detected",
            file=sys.stderr,
        )
        if overall_exit == 0:
            overall_exit = 1

    # Only in-scope gaps gate: a declared covering group ran in this tier but
    # not green. Out-of-scope gaps (covered only by other tiers, or undeclared)
    # are reported but must not fail a tier for coverage it can't produce.
    gaps = [s for s in statuses if s.state == "gap"]
    in_scope_gaps = [s for s in gaps if s.in_scope]
    out_of_scope_gaps = [s for s in gaps if not s.in_scope]
    if out_of_scope_gaps:
        ids = ", ".join(s.path.id for s in out_of_scope_gaps)
        print(
            f"[rig] ℹ️  {len(out_of_scope_gaps)} critical-path gap(s) out of scope "
            f"for this run (warn-only, not covered by any group in this tier): "
            f"{ids}",
            file=sys.stderr,
        )
    if in_scope_gaps and args.fail_on_critical_gap:
        ids = ", ".join(s.path.id for s in in_scope_gaps)
        print(
            f"\n[rig] ⚠️  {len(in_scope_gaps)} critical-path gap(s) detected "
            f"(fail-on-critical-gap): {ids}",
            file=sys.stderr,
        )
        if overall_exit == 0:
            overall_exit = 1

    if args.update_baseline and overall_exit == 0:
        try:
            cp.merge_into_baseline(statuses, args.baseline)
            print(f"[rig] merged into baseline: {args.baseline}")
        except cp.BaselineCorruptError as exc:
            print(f"[rig] ❌ baseline write skipped: {exc}", file=sys.stderr)
            overall_exit = 1

    return overall_exit


# ── execution helpers ─────────────────────────────────────────────────────────


def _blocked_result(group: GroupDefinition, blocked_by: tuple[str, ...]) -> GroupResult:
    """A group that never ran because a dependency failed or was itself blocked."""
    return GroupResult(
        name=group.name,
        tier=group.tier,
        exit_code=0,
        passed=0,
        failed=0,
        errors=0,
        skipped=0,
        duration_seconds=0.0,
        blocked_by=blocked_by,
    )


def _db_env_from_postgres_url(url: str) -> dict[str, str]:
    """Translate a provisioned Postgres URL into the discrete ``DB_*`` vars
    Django reads (``backend/settings/base.py``).

    The rig provisions a throwaway Postgres via testcontainers for groups
    declaring ``requires_services: [postgres]``. Without this translation the
    backend falls back to the compose hostname ``backend-db-1``, unreachable
    from the host-side pytest, and every ``django_db`` test errors on connect.
    """
    # e.g. postgresql+psycopg2://user:pass@host:49153/dbname
    parts = urlsplit(url)
    return {
        "DB_HOST": parts.hostname or "localhost",
        "DB_PORT": str(parts.port or 5432),
        "DB_USER": parts.username or "test",
        "DB_PASSWORD": parts.password or "test",
        "DB_NAME": parts.path.lstrip("/") or "test",
    }


def _inject_infra_env(
    env: dict[str, str],
    group: GroupDefinition,
    endpoints: PlatformEndpoints | None,
) -> None:
    # Postgres/Redis override so a stale shell value can't shadow the
    # provisioned testcontainer; MinIO uses setdefault so a developer's own
    # endpoint wins.
    if endpoints is None:
        return
    infra = endpoints.infra
    if "postgres" in group.requires_services and infra.postgres_url:
        db_env = _db_env_from_postgres_url(infra.postgres_url)
        env.update(db_env)
        # A bare Postgres only has `public`; groups needing another declare it.
        env.setdefault("DB_SCHEMA", "public")
        db_env["DB_SCHEMA"] = env["DB_SCHEMA"]
        # Fixtures read TEST_DB_* so a developer run keeps pointing at their own
        # compose DB. Mirror onto that prefix or those tests connect to the
        # dev-compose defaults instead of the provisioned container. setdefault so
        # a group can still override with its own TEST_DB_* escape hatch.
        for key, value in db_env.items():
            env.setdefault(f"TEST_{key}", value)
    if "minio" in group.requires_services and infra.minio_endpoint:
        # http: this is a local, throwaway testcontainers MinIO with no TLS.
        env.setdefault(
            "MINIO_ENDPOINT_URL",
            f"http://{infra.minio_endpoint}",  # NOSONAR
        )
        if infra.minio_access_key:
            env.setdefault("MINIO_ACCESS_KEY_ID", infra.minio_access_key)
        if infra.minio_secret_key:
            env.setdefault("MINIO_SECRET_ACCESS_KEY", infra.minio_secret_key)
    if "redis" in group.requires_services and infra.redis_host:
        redis_port = str(infra.redis_port or 6379)
        env["REDIS_HOST"] = infra.redis_host
        env["REDIS_PORT"] = redis_port
        env["CELERY_BROKER_BASE_URL"] = f"redis://{infra.redis_host}:{redis_port}"


def _coverage_attesting_groups(results: list[GroupResult]) -> set[str]:
    # "empty" (exit 5) stays non-failing for the build, but a covering group
    # that collected zero tests must not attest critical-path coverage — a
    # broken marker expression would otherwise report ✅ with zero tests run.
    return {r.name for r in results if r.status == STATUS_PASS}


def _marker_proven_paths(
    registry: cp.CriticalPathRegistry,
    group_results: list[GroupResult],
    reports_dir: Path,
) -> tuple[set[str], list[str]]:
    """Union critical-path ids attested by passing marked tests across groups.

    Returns ``(known_ids, unknown_ids)`` — unknown ids (marker typos) must fail
    the build or the marked test silently attests nothing.
    """
    registry_ids = {p.id for p in registry.paths}
    proven: set[str] = set()
    for r in group_results:
        proven |= passed_critical_path_ids(r.name, reports_dir)
    return proven & registry_ids, sorted(proven - registry_ids)


def _is_missing_placeholder(group: GroupDefinition) -> bool:
    if not group.optional:
        return False
    wd = group.absolute_workdir()
    if not wd.exists():
        return True
    return not all(p.exists() for p in group.absolute_paths())


def _execute_group(
    group: GroupDefinition,
    *,
    reports_dir: Path,
    marker: str | None,
    paths_override: str | None,
    coverage: bool,
    parallel: bool,
    workers: str,
    timeout: int,
    endpoints: PlatformEndpoints | None,
) -> tuple[GroupResult | None, int]:
    group_reports = reports_dir / group.name
    group_reports.mkdir(parents=True, exist_ok=True)
    junit = group_reports / "junit.xml"
    md_report = group_reports / "report.md"

    env = _subprocess_env()
    # A group may set its own PYTHONPATH (workdir-relative imports); keep the rig
    # plugin dir on it so `-p rig_critical_path` stays importable instead of being
    # clobbered by a plain env.update.
    group_env = dict(group.env)
    env["PYTHONPATH"] = os.pathsep.join(
        p
        for p in (
            str(_PYTEST_PLUGIN_DIR),
            group_env.pop("PYTHONPATH", None),
            env.get("PYTHONPATH"),
        )
        if p
    )
    env.update(group_env)
    if endpoints is not None:
        env.setdefault("UNSTRACT_BACKEND_URL", endpoints.backend_url)
        env.setdefault("UNSTRACT_FRONTEND_URL", endpoints.frontend_url)
        env.setdefault("UNSTRACT_PROMPT_SERVICE_URL", endpoints.prompt_service_url)
        env.setdefault("UNSTRACT_PLATFORM_SERVICE_URL", endpoints.platform_service_url)
        env.setdefault("UNSTRACT_RUNNER_URL", endpoints.runner_url)
        env.setdefault("UNSTRACT_X2TEXT_URL", endpoints.x2text_url)
        # Stamp the run with a per-invocation sentinel so e2e tests can
        # distinguish "rig brought the platform up" from "stale shell env
        # leaked in". `setdefault` would let a leaked sentinel win, which
        # defeats the purpose — set unconditionally.
        env["UNSTRACT_RIG_SESSION_ID"] = _rig_session_id()
    _inject_infra_env(env, group, endpoints)
    if coverage and group.coverage_source:
        env.update(coverage_env(group.name, reports_dir))

    # Best-effort dep prep. Each `uv` call uses check=False so a transient
    # install failure (e.g. network blip during `uv pip install`) doesn't kill
    # the whole rig; pytest will surface a real missing-module error if so.
    # If you're debugging "ModuleNotFoundError" in a group, scroll up for the
    # uv warnings — they're the smoking gun.
    _prepare_group_env(group, env=env)

    workdir = group.absolute_workdir()

    if group.runner == "hurl":
        cmd = _hurl_command(group, workdir)
        exit_code = _spawn(cmd, env=env, cwd=workdir, timeout=timeout)
        _write_synthetic_junit_safe(junit, group.name, exit_code)
    elif group.runner in ("vitest", "playwright"):
        # Playwright's JUnit path comes from its config, not a flag; the config
        # reads this. Harmless for vitest, which takes --outputFile.
        env.setdefault("PLAYWRIGHT_JUNIT_OUTPUT_NAME", str(junit))
        cmd = _node_command(group, junit=junit)
        exit_code = _spawn(cmd, env=env, cwd=workdir, timeout=timeout)
        # Both runners write real JUnit themselves, so there is nothing to
        # synthesise on the happy path. A crash before the reporter flushes
        # (missing node_modules, a config error) leaves no file at all, which
        # `parse_junit` reports as a missing group rather than a failure — so
        # backfill one to keep a red run visible in the summary.
        if not junit.exists():
            _write_synthetic_junit_safe(junit, group.name, exit_code)
    elif (isolation_error := _config_isolation_error(group, workdir)) is not None:
        print(isolation_error, file=sys.stderr)
        # 4 is pytest's usage-error code, so this gates like any hard failure.
        exit_code = 4
        _write_synthetic_junit_safe(junit, group.name, exit_code)
    else:
        cmd = _pytest_command(
            group,
            workdir=workdir,
            junit=junit,
            md_report=md_report,
            marker=marker,
            paths_override=paths_override,
            coverage=coverage,
            parallel=parallel,
            workers=workers,
            timeout=timeout,
        )
        exit_code = _spawn(cmd, env=env, cwd=workdir, timeout=timeout)

    try:
        (group_reports / "exit.txt").write_text(str(exit_code))
    except OSError as err:
        print(
            f"[rig] could not write exit.txt for {group.name}: {err}",
            file=sys.stderr,
        )

    return parse_junit(group.name, group.tier, reports_dir), exit_code


# Injected into every group's ephemeral `uv run` env via --with, so they
# survive the per-call venv re-sync (unlike `uv pip install`). Includes pytest
# and requests themselves — needed regardless of the workdir project's deps.
RIG_PYTEST_PLUGINS = (
    "pytest>=8.0.1",
    "pytest-md-report>=0.6.2",
    "pytest-timeout>=2.3.1",
    "pytest-xdist>=3.5.0",
    "pytest-cov>=4.1.0",
    "pytest-mock>=3.12.0",
    "requests>=2.31.0",
    "minio>=7.2.0",
)


def _prepare_group_env(group: GroupDefinition, *, env: dict[str, str]) -> None:
    """Sync deps for a group. Mirrors what the old per-service tox envs did,
    and ensures the rig's pytest plugins are present in the group's venv.
    """
    workdir = group.absolute_workdir()
    if not shutil.which("uv"):
        return
    if group.uv_sync_group:
        subprocess.run(
            ["uv", "sync", "--group", group.uv_sync_group],
            cwd=workdir,
            env=env,
            check=False,
        )
    # install_editable is handled in _pytest_command via `--with-editable`;
    # installing it here would be wiped by `uv run`'s venv re-sync.
    if group.pip_install:
        subprocess.run(
            ["uv", "pip", "install", *group.pip_install],
            cwd=workdir,
            env=env,
            check=False,
        )
    # NOTE: rig pytest plugins (pytest-timeout, pytest-md-report, etc.) are
    # injected via `uv run --with ...` in _pytest_command, not installed here.
    # That avoids losing them on the next `uv run` (which re-syncs the venv).


def _run_migrations(spec: MigrateSpec, *, postgres_url: str) -> int:
    """Apply Django migrations to the rig-provisioned database.

    Runs once, right after the container is up and before any group, because the
    schema is a property of the database rather than of a group. Only the
    migrating project's own Django settings are layered over the connection env
    — a group's environment is irrelevant here and may not even exist yet.
    """
    env = {
        **os.environ,
        # spec.env carries Django settings (e.g. schema); the provisioned
        # connection is layered on top so a stray DB_HOST/DB_NAME in spec.env
        # can't silently redirect migrations away from the container the groups
        # then test against.
        **spec.env,
        **_db_env_from_postgres_url(postgres_url),
        # A rig-provisioned Postgres is bare; `public` is the only schema it has,
        # unless the spec deliberately migrates into another.
        "DB_SCHEMA": spec.env.get("DB_SCHEMA", "public"),
    }
    workdir = spec.absolute_workdir()
    base = ["uv", "run", "python"] if shutil.which("uv") else [sys.executable]
    cmd = [*base, "manage.py", "migrate", *spec.apps, "--noinput"]
    print(
        f"[rig] migrating provisioned database via {spec.workdir}: "
        f"{' '.join(spec.apps) or 'all apps'}"
    )
    return _spawn(cmd, env=env, cwd=workdir, timeout=_MIGRATE_TIMEOUT_SECONDS)


def _write_synthetic_junit_safe(path: Path, group_name: str, exit_code: int) -> None:
    """Mirror the exit.txt write's defensive handling: a read-only reports dir
    or full disk shouldn't abort the whole run and orphan completed groups
    before the summary renders.
    """
    try:
        _write_synthetic_junit(path, group_name, exit_code)
    except OSError as err:
        print(
            f"[rig] could not write synthetic junit for {group_name}: {err}",
            file=sys.stderr,
        )


def _pytest_base_cmd(group: GroupDefinition, workdir: Path) -> list[str]:
    if not shutil.which("uv"):
        return [sys.executable, "-m", "pytest"]
    # `uv run` re-syncs the venv each call, wiping anything from `uv pip
    # install`. `--with`/`--with-editable` inject plugins + the project into the
    # ephemeral run env instead, surviving the sync.
    with_args: list[str] = []
    for spec in RIG_PYTEST_PLUGINS:
        with_args += ["--with", spec]
    if group.install_editable:
        with_args += ["--with-editable", str(workdir)]
    return ["uv", "run", *with_args, "pytest"]


# Order matters: pytest picks the first of these it finds walking upward.
_PYTEST_CONFIG_FILES = (
    ("pytest.ini", None),
    (".pytest.ini", None),
    ("pyproject.toml", "[tool.pytest.ini_options]"),
    ("tox.ini", "[pytest]"),
    ("setup.cfg", "[tool:pytest]"),
)


def _resolve_pytest_configfile(workdir: Path) -> Path | None:
    """Find the config file pytest would adopt when invoked from `workdir`.

    Mirrors pytest's upward search. `pytest.ini` counts even when empty; the
    shared files only count when they carry a pytest section.
    """
    for directory in [workdir, *workdir.parents]:
        for name, required_section in _PYTEST_CONFIG_FILES:
            candidate = directory / name
            if not candidate.is_file():
                continue
            if required_section is None:
                return candidate
            try:
                lines = candidate.read_text(encoding="utf-8").splitlines()
            except OSError:
                continue
            # Match the section header on its own line, as the TOML/INI parsers
            # do — a commented-out or docstring mention is not a real config.
            if any(line.strip() == required_section for line in lines):
                return candidate
        if directory == REPO_ROOT:
            break
    return None


def _config_isolation_error(group: GroupDefinition, workdir: Path) -> str | None:
    """Reject a group that would silently inherit an unrelated project's config.

    `cd workdir && pytest <paths>` does not pin config to workdir — pytest walks
    up from the common ancestor of its path args. A nested project without its
    own config therefore adopts its parent's `addopts` (coverage targets,
    --strict-config), which fails in ways that have nothing to do with the
    group's tests. Inheriting the repo-root config is the normal monorepo case
    and stays allowed; inheriting some *other* project's config never is.
    """
    # Match pytest: ini discovery starts at the common ancestor of the path args
    # (the rig always passes paths), not the workdir — they differ whenever paths
    # reach into a subdirectory that is itself a nested project.
    paths = [str(workdir / p) for p in group.paths]
    start = Path(os.path.commonpath(paths)) if paths else workdir
    configfile = _resolve_pytest_configfile(start)
    if configfile is None or configfile.parent in (workdir, REPO_ROOT):
        return None
    # An empty pytest.ini is the lightest fix for a workdir with no pyproject.
    fix_target = (
        workdir / "pyproject.toml"
        if (workdir / "pyproject.toml").is_file()
        else workdir / "pytest.ini"
    )
    return (
        f"[rig] group '{group.name}' would run under {configfile}, which belongs "
        f"to neither its workdir ({workdir}) nor the repo root.\n"
        f"[rig] pytest resolves config by walking up from the tests' common "
        f"ancestor, so this group inherits that project's addopts.\n"
        f"[rig] fix: give the group its own pytest config, e.g. add a "
        f"[tool.pytest.ini_options] section to or create {fix_target}."
    )


def _pytest_command(
    group: GroupDefinition,
    *,
    workdir: Path,
    junit: Path,
    md_report: Path,
    marker: str | None,
    paths_override: str | None,
    coverage: bool,
    parallel: bool,
    workers: str,
    timeout: int,
) -> list[str]:
    base = _pytest_base_cmd(group, workdir)

    cmd = [
        *base,
        "-v",
        f"--junitxml={junit}",
        f"--timeout={timeout}",
        # Marker→junit-property plugin (dir added to PYTHONPATH in
        # _execute_group). junit_family=legacy: the default xunit2 schema
        # rejects per-testcase <properties>, which the coverage attestation
        # reads.
        "-p",
        "rig_critical_path",
        "-o",
        "junit_family=legacy",
    ]
    # pytest-md-report does not aggregate worker output reliably under xdist.
    # Emit markdown only on serial runs; junit + reporting.py's _render_markdown
    # cover the parallel case.
    if not parallel:
        cmd += [
            "--md-report",
            "--md-report-flavor=gfm",
            f"--md-report-output={md_report}",
        ]
    if coverage and group.coverage_source:
        cmd += [
            f"--cov={group.coverage_source}",
            "--cov-report=",
        ]
    if parallel:
        cmd += ["-n", workers]
    effective_marker = marker or group.markers
    if effective_marker:
        cmd += ["-m", effective_marker]
    cmd += list(group.pytest_extra)

    if paths_override:
        cmd += [p.strip() for p in paths_override.split(",") if p.strip()]
    else:
        # Paths are relative to workdir so pytest runs as `cd workdir && pytest <path>`.
        for p in group.paths:
            cmd.append(p)
    return cmd


def _node_command(
    group: GroupDefinition,
    *,
    junit: Path,
) -> list[str]:
    """Build the command for a Node-based group (`vitest` / `playwright`).

    Both write JUnit natively, so the rig points their reporter straight at the
    same `junit.xml` every other group writes and `parse_junit` reads it
    unchanged — vitest's `<testsuites>` root is already the shape it expects.

    `npx --no-install` on purpose: silently downloading a runner mid-run would
    turn a missing dependency into a slow, network-dependent surprise. Without
    `node_modules` this exits non-zero and the caller backfills a red junit.
    """
    if not shutil.which("npx"):
        # Same convention as `_hurl_command`'s empty-file case: exit 5 reads as
        # "nothing collected" rather than a failure, so a machine without Node
        # does not turn the whole run red.
        print(
            f"[rig] npx not found; skipping {group.name} (runner={group.runner})",
            file=sys.stderr,
        )
        return ["sh", "-c", "exit 5"]

    if group.runner == "vitest":
        cmd = [
            "npx",
            "--no-install",
            "vitest",
            "run",
            "--reporter=junit",
            f"--outputFile={junit}",
        ]
        # `paths` are optional for vitest: with none it runs the whole suite,
        # which is what the `frontend` group wants.
        cmd += [str(p) for p in group.paths if p not in (".", "")]
        return cmd

    # Playwright takes its reporter output path from config, not a CLI flag, so
    # the config reads PLAYWRIGHT_JUNIT_OUTPUT_NAME — which the rig exports in
    # `_group_env`. `--timeout` is per-test milliseconds, distinct from the
    # rig's whole-group timeout, so it is deliberately not derived from it.
    cmd = ["npx", "--no-install", "playwright", "test"]
    cmd += [str(p) for p in group.paths if p not in (".", "")]
    return cmd


def _hurl_command(group: GroupDefinition, workdir: Path) -> list[str]:
    files: list[str] = []
    for p in group.paths:
        target = workdir / p
        if target.is_dir():
            files.extend(sorted(str(f) for f in target.rglob("*.hurl")))
        elif target.is_file():
            files.append(str(target))
    if not files:
        # Surface as exit 5 ("no tests collected") for consistency with pytest.
        return ["sh", "-c", "exit 5"]
    return ["hurl", "--test", *files]


def _write_synthetic_junit(path: Path, group_name: str, exit_code: int) -> None:
    """Synthesise a JUnit XML for a group pytest never wrote one for (hurl runs,
    or a group rejected by the config-isolation guardrail).

    Exit 5 ("no tests collected") must produce failures=0; otherwise an empty
    hurl group would show ⚪ via :class:`GroupResult` while also being counted
    as a failure in totals + critical-path evaluation.

    ``group_name`` is XML-escaped: a group key containing ``"``/``&``/``<``
    would otherwise produce malformed XML, which ``parse_junit`` then reads as
    a phantom error on a green hurl run.
    """
    is_failure = exit_code != 0 and exit_code != 5
    failures = 1 if is_failure else 0
    failure_tag = f'<failure message="hurl exit {exit_code}"/>' if is_failure else ""
    name = saxutils.escape(group_name, _XML_ATTR_ESCAPES)
    path.write_text(
        '<?xml version="1.0" encoding="UTF-8"?>\n'
        f'<testsuite name="{name}" tests="1" failures="{failures}" '
        f'errors="0" skipped="0" time="0">\n'
        f'  <testcase classname="{name}" name="hurl-suite">{failure_tag}'
        f"</testcase>\n"
        f"</testsuite>\n"
    )


def _write_blocked_junit(
    path: Path, group_name: str, blocked_by: tuple[str, ...]
) -> None:
    """Synthesise a junit for a group that never ran because a dependency did
    not pass.

    Carries the blocking group names as a property (zero test counters) so CI's
    `rig report`, which rebuilds results purely from junit, renders the ⏭️
    blocked row instead of dropping the group.
    """
    name = saxutils.escape(group_name, _XML_ATTR_ESCAPES)
    value = saxutils.escape(",".join(blocked_by), _XML_ATTR_ESCAPES)
    path.write_text(
        '<?xml version="1.0" encoding="UTF-8"?>\n'
        f'<testsuite name="{name}" tests="0" failures="0" errors="0" '
        'skipped="0" time="0">\n'
        "  <properties>\n"
        f'    <property name="rig-blocked-by" value="{value}"/>\n'
        "  </properties>\n"
        "</testsuite>\n"
    )


def _persist_blocked_group(
    reports_dir: Path, group: GroupDefinition, blocked_by: tuple[str, ...]
) -> None:
    """Write a blocked group's junit + exit.txt so it survives the artifact
    round-trip into CI's `rig report`. Best-effort: a read-only reports dir
    shouldn't abort the run.
    """
    group_reports = reports_dir / group.name
    try:
        group_reports.mkdir(parents=True, exist_ok=True)
        _write_blocked_junit(group_reports / "junit.xml", group.name, blocked_by)
        (group_reports / "exit.txt").write_text("0")
    except OSError as err:
        print(
            f"[rig] could not persist blocked group {group.name}: {err}",
            file=sys.stderr,
        )


def _spawn(cmd: list[str], *, env: dict[str, str], cwd: Path, timeout: int) -> int:
    start = time.monotonic()
    try:
        result = subprocess.run(cmd, cwd=cwd, env=env, timeout=timeout + 30)
        return result.returncode
    except subprocess.TimeoutExpired:
        print(
            f"[rig] TIMEOUT after {time.monotonic() - start:.0f}s: {' '.join(cmd)}",
            file=sys.stderr,
        )
        return 124
    except FileNotFoundError as exc:
        print(f"[rig] command not found: {exc}", file=sys.stderr)
        return 127