A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
reuse build artifacts in the beta flow and filter tests by path (#3187)
kt776c911ecba4
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a82e3371b8..8f6d720c54 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -17,9 +17,29 @@# #2627's fork_from test-target errors land as latent compile breaks, and# #2635's login-recovery test land red — none noticed by CI.#-# pull_request against main/rust — the same gates on the merge candidate,+# pull_request against main — the same gates on the merge candidate,# so the merge bar stops relying solely on review bots.#+# merge_group — full workspace tests on the queued merge candidate,+# not the PR's narrowed selection. Branch protection must separately+# enable merge queue and require the final gates.+#+# THE PR SMOKE MODEL (the touched-crate path filter): the PR wave narrows+# the TEST phase to touched crates and their reverse dependents' units+# (the `changes` job maps the PR's changed files to crates; any path+# outside crates/<pkg>/** fails safe+# to the full selection — CI/scripts/vendor/runtime changes affect the+# whole suite, and docs-only waste is cheaper than a silent hole). The+# stable crc32 shard assignment is untouched — only the unit selection+# narrows — and the push wave keeps running the full union, so main is the+# authority: the PR proves the touched crates' runtime behavior fast, the+# merge proves the whole workspace. fmt + check/clippy stay full-scope on+# PRs (the compile authority); reverse-dep selection also runs tests of+# crates affected by a dependency's behavior change. The shard manifests+# record the selection scope and the summary job audits the union against it, so a+# narrowed run can never masquerade as a full one and coverage can never+# silently regress.+## The test phase is sharded: `cargo test --workspace` ran every test binary# serially in one job (~13 minutes; the per-PR wall-clock floor). It is now# the union of SHARD_COUNT parallel shard jobs: scripts/ci_test_shard.py enumerates@@ -49,15 +69,17 @@ on:pull_request:types: [opened, synchronize, reopened, ready_for_review]branches: [main]+ merge_group:+ types: [checks_requested]permissions:contents: readconcurrency:group: ci-${{ github.workflow }}-${{ github.ref }}- # PRs: a newer push supersedes the stale gates. Pushes: the newest tip- # supersedes the older run — under this fleet's merge cadence a queued- # (non-canceling) push run would lag hours behind and bill stale tips.+ # PRs: a newer push supersedes stale gates. Queue entries use their own+ # merge-group ref; a superseded entry's gates can be cancelled. Pushes:+ # the newest tip supersedes the older run instead of billing stale tips.cancel-in-progress: trueenv:@@ -82,13 +104,62 @@ env:SHARD_COUNT: "8"jobs:+ # The touched-crate path filter (the PR smoke's selection). Push and queue+ # runs use the full selection; PR runs select crates plus reverse deps,+ # failing safe to the full selection on any non-crate path. The output+ # feeds test shards' --crates; empty means "the full union".+ changes:+ name: touched crates (PR) / full selection (push + queue)+ runs-on: ubuntu-latest+ timeout-minutes: 10+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft+ outputs:+ crates: ${{ steps.select.outputs.crates }}+ scope: ${{ steps.select.outputs.scope }}+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false+ - name: Select the touched crates+ id: select+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ run: |+ set -euo pipefail+ if [ "${{ github.event_name }}" != "pull_request" ]; then+ echo "${{ github.event_name }} wave: the full selection"+ echo "crates=" >> "${GITHUB_OUTPUT}"+ echo "scope=all" >> "${GITHUB_OUTPUT}"+ exit 0+ fi+ # The PR's exact change set: the pulls files API (merge-base+ # semantics, renames carrying both sides), fully paginated —+ # `gh pr view --json files` caps at 100 rows and would silently+ # under-map a large PR, and its field names differ from the REST+ # rows (`path` vs `filename`). The mapping itself is fail-safe+ # by construction (scripts/ci_pr_crates.py + its battery): any+ # non-crate path, a rename's removed side, an empty column, or+ # the API's 3000-file ceiling means the FULL selection runs.+ gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \+ --paginate --jq '.[] | [.filename, .previous_filename // ""] | @tsv' \+ > "${RUNNER_TEMP}/pr-files.tsv"+ python3 scripts/ci_pr_crates.py "${RUNNER_TEMP}/pr-files.tsv"+ - name: Report the selection+ run: |+ set -euo pipefail+ echo "wave selection: ${{ steps.select.outputs.scope }}"+ if [ -n "${{ steps.select.outputs.crates }}" ]; then+ echo "crates under test: ${{ steps.select.outputs.crates }}"+ fi+fmt:name: fmtruns-on: ubuntu-24.04timeout-minutes: 30# Draft PRs do not gate (no billing on work-in-progress pushes); the# ready_for_review trigger runs the gates once a draft is marked ready.- if: github.event_name == 'push' || !github.event.pull_request.draft+ if: github.event_name != 'pull_request' || !github.event.pull_request.draftsteps:- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2with:@@ -103,7 +174,7 @@ jobs:name: check + clippy (workspace)runs-on: ubuntu-24.04timeout-minutes: 45- if: github.event_name == 'push' || !github.event.pull_request.draft+ if: github.event_name != 'pull_request' || !github.event.pull_request.draftsteps:- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2with:@@ -135,12 +206,16 @@ jobs:# ONLY its test subset. The union of the shards is exactly the old serial# `cargo test --workspace`.test-shard:+ # The shard legs need the wave's selection (the changes job's crates+ # output) before they can decide what to run — and whether the+ # sibling-binary build is needed at all.+ needs: [changes]# strategy.job-total (the matrix leg count) auto-derives the displayed# total; the env context is not reachable from a job name.name: test shard ${{ matrix.shard }}/${{ strategy.job-total }}runs-on: ubuntu-24.04timeout-minutes: 45- if: github.event_name == 'push' || !github.event.pull_request.draft+ if: github.event_name != 'pull_request' || !github.event.pull_request.draftstrategy:# One red shard must not cancel the others: their own conclusions are# the per-shard signal the summary merges.@@ -168,18 +243,30 @@ jobs:run: |python3 -m pip install --user uvecho "$HOME/.local/bin" >> "$GITHUB_PATH"+ # The wave's selection, resolved once per leg before anything builds:+ # the narrowed selection's unit list decides whether the bins build+ # below has any work to do (an empty selection runs no units and needs+ # no sibling binaries).+ - name: Resolve this shard's selection+ id: selection+ run: |+ set -euo pipefail+ python3 scripts/ci_test_shard.py --shard ${{ matrix.shard }} --total ${{ env.SHARD_COUNT }} --crates "${{ needs.changes.outputs.crates }}" --print-selection# Sibling-binary parity: the serial `cargo test --workspace` built every# workspace bin into the shared profile dir as a side effect, and the# e2e suites look each other's binaries up there (pa-cli's# daemon_commands_e2e spawns pa-daemon this way). A shard builds its# test subset only, so the bins are built explicitly once up front —# first measured on run 36085993711, where the missing pa-daemon bin- # failed exactly this suite.+ # failed exactly this suite. A leg with no selected units skips this+ # (and runs nothing below); the push wave always selects units, so its+ # behavior is unchanged.- name: Build the workspace bins (sibling-binary parity for the e2e suites)+ if: steps.selection.outputs.units != '0'run: cargo build --locked --workspace --bins- name: Run this shard's test subsetrun: |- python3 scripts/ci_test_shard.py --shard ${{ matrix.shard }} --total ${{ env.SHARD_COUNT }} --manifest "$RUNNER_TEMP/shard-manifest-${{ matrix.shard }}.json"+ python3 scripts/ci_test_shard.py --shard ${{ matrix.shard }} --total ${{ env.SHARD_COUNT }} --crates "${{ needs.changes.outputs.crates }}" --manifest "$RUNNER_TEMP/shard-manifest-${{ matrix.shard }}.json"- name: Upload the shard manifest (always — the summary audits it)if: always()uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2@@ -196,7 +283,7 @@ jobs:runs-on: ubuntu-24.04timeout-minutes: 10needs: [test-shard]- if: always() && (github.event_name == 'push' || !github.event.pull_request.draft)+ if: always() && (github.event_name != 'pull_request' || !github.event.pull_request.draft)steps:- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2with:diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.ymlindex 108ba1016e..dabe197622 100644--- a/.github/workflows/continuous.yml+++ b/.github/workflows/continuous.yml@@ -200,6 +200,16 @@ jobs:--catalog-assets "target/${{ matrix.target }}/catalog-assets" \--out-dir "target/${{ matrix.target }}/dist"cat "target/${{ matrix.target }}/dist/manifest.json"+ - name: Livecheck the assembled continuous artifact on its native runner+ run: |+ set -euo pipefail+ version="$(sed -n 's/^version *= *"\([^"]*\)".*/\1/p' Cargo.toml | head -1)"+ [[ -n "${version}" ]] || { echo "could not read the workspace version"; exit 1; }+ python3 scripts/release/verify_release.py \+ --dist-dir "target/${{ matrix.target }}/dist" \+ --version "${version}" \+ --target "${{ matrix.target }}" \+ --sha "${GITHUB_SHA}"- name: Upload artifactsuses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0with:@@ -262,6 +272,16 @@ jobs:--catalog-assets "target/${{ matrix.target }}/catalog-assets" \--out-dir "target/${{ matrix.target }}/dist"cat "target/${{ matrix.target }}/dist/manifest.json"+ - name: Livecheck the assembled continuous artifact on its native runner+ run: |+ set -euo pipefail+ version="$(sed -n 's/^version *= *"\([^"]*\)".*/\1/p' Cargo.toml | head -1)"+ [[ -n "${version}" ]] || { echo "could not read the workspace version"; exit 1; }+ python3 scripts/release/verify_release.py \+ --dist-dir "target/${{ matrix.target }}/dist" \+ --version "${version}" \+ --target "${{ matrix.target }}" \+ --sha "${GITHUB_SHA}"- name: Upload artifactsuses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0with:diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex b3f1aa8416..0d2b97621a 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -1,5 +1,25 @@# Release pipeline for Prime Agent (Rust).#+# THE TWO BUILD ROUTES (one promote gate):+# - a plain vX.Y.Z tag (stable) BUILDS its own 4-target matrix: the stable+# cadence is rare, and its cut must not depend on a continuous run the+# staleness gate may have skipped (a superseded push builds nothing) or+# artifact retention may have evicted;+# - a vX.Y.Z-beta* tag (nightly) CONSUMES the continuous build's+# artifacts: continuous already built the exact parent commit the+# nightly tag names (nightly.yml waits out that run's green build+# matrix before cutting the tag), so rebuilding here would pay the+# full matrix twice per green merge for a tree that differs from its+# parent only by the version rows. The reuse-continuous job downloads+# that run's workflow artifacts (the download-by-run route the+# installer's old bootstrap used) and restamps them to the beta+# version — packaging only, no rebuild; the built bytes stay the+# continuous build's (the restamp re-proves what it can on the runner+# and asserts the continuous commit; the rest is continuous's own+# green gates on that commit). Both routes upload the same+# artifacts-<target> set into this run, so the promote gate consumes+# one shape from either route.+#name: release@@ -59,6 +79,9 @@ jobs:build-gnu:needs: tag-check+ # Stable-only: a -beta* tag consumes the continuous build instead (the+ # reuse-continuous job); exactly one of the two routes runs per tag.+ if: ${{ !contains(github.ref_name, '-') }}name: build-${{ matrix.target }}runs-on: ${{ matrix.runs-on }}# GLIBC 2.35 build baseline: artifacts built on the ubuntu-24.04 runner@@ -197,6 +220,9 @@ jobs:build-darwin:needs: tag-check+ # Stable-only: a -beta* tag consumes the continuous build instead (the+ # reuse-continuous job); exactly one of the two routes runs per tag.+ if: ${{ !contains(github.ref_name, '-') }}name: build-${{ matrix.target }}runs-on: ${{ matrix.runs-on }}permissions:@@ -259,8 +285,136 @@ jobs:target/${{ matrix.target }}/dist/manifest.jsonif-no-files-found: error+ # THE BETA CHANNEL'S BUILD (the consume route): the continuous workflow+ # already built the exact commit this beta names - nightly.yml waits out+ # the pushed commit's continuous run (its whole build matrix green) before+ # cutting the tag, so the tag-only commit's parent is a proven-built tip.+ # This job resolves that parent, downloads ITS continuous run's workflow+ # artifacts (the download-by-run route the installer's old bootstrap used),+ # and restamps them to the beta version (scripts/release/restamp_continuous.py):+ # the built bytes stay the continuous build's; the version-shaped facts move+ # - the inner package.json (the runtime version manifest the shipped binary+ # reports; the continuous stamp <version>-continuous.<sha> becomes the beta+ # version), the archive/decoder names, and the manifest/SHA256SUMS rows (the+ # manifest's `commit` provenance is KEPT). The result is re-uploaded as THIS+ # run's artifacts-<target> set, so the promote gate consumes the same shape+ # from either route. The gates the release build jobs ran on their own+ # binaries (livecheck, GLIBC baseline) re-run here for the runner's own+ # target; the other targets' evidence is the continuous run's own green+ # build jobs on this exact commit (asserted via the manifest's commit row).+ reuse-continuous:+ needs: tag-check+ if: ${{ contains(github.ref_name, '-') }}+ name: consume the continuous build (no rebuild)+ runs-on: ubuntu-latest+ permissions:+ contents: read+ # Cross-run workflow-artifact download: the continuous run's set.+ actions: read+ timeout-minutes: 30+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false+ - name: Resolve the continuous run that built this beta's parent+ id: resolve+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ set -euo pipefail+ # THE ANNOTATED-TAG PEEL (the same ref-chain resolution the promote+ # job's publish step uses): the tag names the tag-only commit+ # nightly.yml created; its first parent is the continuous-built tip.+ tag="${GITHUB_REF_NAME}"+ ref_json="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${tag}")"+ obj_type="$(printf '%s' "${ref_json}" | jq -r '.object.type')"+ obj_sha="$(printf '%s' "${ref_json}" | jq -r '.object.sha')"+ if [ "${obj_type}" = "tag" ]; then+ tag_obj="$(gh api "repos/${GITHUB_REPOSITORY}/git/tags/${obj_sha}")"+ obj_sha="$(printf '%s' "${tag_obj}" | jq -r '.object.sha')"+ fi+ parent="$(gh api "repos/${GITHUB_REPOSITORY}/git/commits/${obj_sha}" --jq '.parents[0].sha')"+ # The run whose artifacts this beta consumes - the same query+ # nightly.yml resolves the parent with. The cut is only created+ # after that run concluded green with its whole build matrix,+ # so the run must exist and be green here; anything else is a+ # malformed tag (fail loudly, never fall back to a rebuild).+ runs_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/continuous.yml/runs?branch=main&per_page=1&head_sha=${parent}" 2>/dev/null || echo '{"workflow_runs":[]}')"+ read -r status conclusion run_id < <(printf '%s\n' "${runs_json}" | jq -r \+ '(.workflow_runs[0] // {}) | [(.status // "absent"), (.conclusion // "-"), (.id // "-")] | @tsv')+ if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then+ echo "::error::no successful continuous run for ${parent} (${status}/${conclusion}): the beta cannot consume artifacts that do not exist - resolve the continuous run, then re-run this release"+ exit 1+ fi+ echo "beta parent (the continuous-built commit): ${parent}"+ echo "continuous run ${run_id} built ${parent}"+ echo "parent=${parent}" >> "${GITHUB_OUTPUT}"+ echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"+ - name: Download the continuous build's artifacts (by run id)+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ run-id: ${{ steps.resolve.outputs.run-id }}+ github-token: ${{ github.token }}+ pattern: artifacts-*+ path: incoming+ - name: Restamp the continuous set to the beta version (no rebuild)+ run: |+ set -euo pipefail+ # The promote gate consumes exactly the 4-target set; a partial+ # download must fail here, not at the channel manifest.+ expected="artifacts-aarch64-apple-darwin artifacts-aarch64-unknown-linux-gnu artifacts-x86_64-apple-darwin artifacts-x86_64-unknown-linux-gnu"+ actual="$(find incoming -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort | tr '\n' ' ')"+ if [ "${actual% }" != "${expected}" ]; then+ echo "::error::the continuous run's artifact set is incomplete: ${actual% }"+ exit 1+ fi+ python3 scripts/release/restamp_continuous.py \+ --incoming incoming \+ --version "${RELEASE_VERSION#v}" \+ --expect-commit "${{ steps.resolve.outputs.parent }}" \+ --out-dir restamped+ - name: Upload the restamped set as this run's build artifacts+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0+ with:+ name: artifacts-aarch64-apple-darwin+ path: restamped/artifacts-aarch64-apple-darwin/*+ if-no-files-found: error+ - name: Upload the restamped set as this run's build artifacts (aarch64-unknown-linux-gnu)+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0+ with:+ name: artifacts-aarch64-unknown-linux-gnu+ path: restamped/artifacts-aarch64-unknown-linux-gnu/*+ if-no-files-found: error+ - name: Upload the restamped set as this run's build artifacts (x86_64-apple-darwin)+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0+ with:+ name: artifacts-x86_64-apple-darwin+ path: restamped/artifacts-x86_64-apple-darwin/*+ if-no-files-found: error+ - name: Upload the restamped set as this run's build artifacts (x86_64-unknown-linux-gnu)+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0+ with:+ name: artifacts-x86_64-unknown-linux-gnu+ path: restamped/artifacts-x86_64-unknown-linux-gnu/*+ if-no-files-found: error+promote:- needs: [build-gnu, build-darwin]+ # One gate for both build routes: the stable build jobs (plain tags) or+ # the reuse-continuous set (beta tags) upload the same artifacts-<target>+ # set into this run. THE ROUTE GATE IS EXPLICIT: a job that needs a+ # SKIPPED job is itself skipped unless its own `if` says otherwise (the+ # GitHub documented default), so the default needs gate would strand+ # promote on BOTH routes - each route deliberately skips the other two+ # legs. The expression admits exactly the two healthy routes and nothing+ # else: (stable) both build jobs succeeded, or (beta) the consume route+ # succeeded; a failed leg, a cancelled run, or a failed tag-check leaves+ # promote skipped, matching the pre-consume failure semantics. The+ # artifact completeness assertion inside the gate is the belt behind the+ # routing: an incomplete set fails the promotion loudly either way.+ if: ${{ !cancelled() && (+ (needs.build-gnu.result == 'success' && needs.build-darwin.result == 'success')+ || needs.reuse-continuous.result == 'success') }}+ needs: [build-gnu, build-darwin, reuse-continuous]name: promotion gate -> release attachruns-on: ubuntu-latestenvironment: release@@ -341,20 +495,32 @@ jobs:import json, pathlib, tarfileincoming = pathlib.Path("incoming")merged, decoders, sums, version = [], [], [], None+ commits = set()for entry_dir in sorted(p for p in incoming.iterdir() if p.is_dir()):manifest = json.loads((entry_dir / "manifest.json").read_text())version = manifest["version"]merged.extend(manifest["binaries"])decoders.extend(manifest.get("decoders", []))+ if manifest.get("commit"):+ commits.add(manifest["commit"])sums.extend(line for line in(entry_dir / "SHA256SUMS").read_text().splitlines()if line.strip())+ # The consume route (beta) stamps every per-target manifest with the+ # continuous commit it built; a set that names more than one commit+ # is a broken restamp and must never merge into one release.+ if len(commits) > 1:+ raise SystemExit(f"per-target manifests name different commits: {sorted(commits)}")merged.sort(key=lambda b: b["file"])decoders.sort(key=lambda d: d["file"])sums.sort(key=lambda line: line.split(None, 1)[1])out = pathlib.Path("release-out")out.mkdir()result = {"version": version, "binaries": merged}+ if commits:+ # The build provenance rides the merged manifest (the stable+ # rebuild route carries no commit row, as before).+ result["commit"] = sorted(commits)[0]if decoders:result["decoders"] = decoders(out / "manifest.json").write_text(diff --git a/Makefile b/Makefileindex d8941a1458..0b87aa7276 100644--- a/Makefile+++ b/Makefile@@ -165,4 +165,19 @@ catalog-assets-gates:fold-gates:python3 scripts/release/test_fold_changelog.py-.PHONY: check deny windows-cross actionlint perf-wave glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates+# The consume route's restamp battery (release.yml's beta channel): the+# continuous artifacts restamped to the beta version keep the built bytes+# and carry the release shape - the provenance case is first (a foreign+# commit's artifacts are refused).+restamp-gates:+ python3 scripts/release/test_restamp.py++# The CI shard tooling's contract battery (ci.yml's PR smoke): the stable+# crc32 assignment under the narrowed selection, the scope-aware summary+# audit, the selection resolver the conditional bins build reads, and the+# fail-safe PR-files mapping the changes job feeds it.+shard-gates:+ python3 scripts/test_ci_test_shard.py+ python3 scripts/test_ci_pr_crates.py++.PHONY: check deny windows-cross actionlint perf-wave glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gatesdiff --git a/scripts/ci_pr_crates.py b/scripts/ci_pr_crates.pynew file mode 100644index 0000000000..c96887361a--- /dev/null+++ b/scripts/ci_pr_crates.py@@ -0,0 +1,144 @@+#!/usr/bin/env python3+"""Map a pull request's changed files to the crates its PR wave tests.++ci.yml's `changes` job resolves the selection: the test phase narrows to the+touched crates and their reverse dependencies (scripts/ci_test_shard.py+--crates) on PRs; push and merge-queue waves run the full union. The+mapping is fail-safe by construction: a selection that cannot be made is+the FULL selection — a wrong guess can+only cost runner minutes, never coverage.++The input is the pull-requests files API (`GET /repos/{o}/{r}/pulls/{n}/files`,+paginated by gh), one TSV row per file: `<filename>` TAB `<previous_filename>`+(the rename sidecar; empty for ordinary rows). The API's file field is named+`filename` (NOT `path` — the `gh pr view --json files` shape differs, and its+100-row cap silently under-maps large PRs).++The output is the job outputs (`crates` — a comma list, empty for the full+selection — and `scope`, the human-readable decision) written to the file in+`GITHUB_OUTPUT` (a plain report otherwise), plus a per-path report on stdout.++Usage (ci.yml):++ gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR}/files" \+ --paginate --jq '.[] | [.filename, .previous_filename // ""] | @tsv' \+ > "${RUNNER_TEMP}/pr-files.tsv"+ python3 scripts/ci_pr_crates.py "${RUNNER_TEMP}/pr-files.tsv"++Tests: python3 scripts/test_ci_pr_crates.py (make shard-gates runs it)+"""++from __future__ import annotations++import os+import sys+from pathlib import Path++# The pulls/files API's own page ceiling: at or beyond it the list is+# truncated and the mapping is not trustworthy, so the full selection runs.+FILES_API_CEILING = 3000++# Conservative CI impact graph: some edges cover behavior beyond a direct+# Cargo.toml link. Over-selecting is safe; missing a dependent test is not.+# The changes job must select without installing Rust or running Cargo.+WORKSPACE_DEPS = {+ "pa-types": set(),+ "pa-telemetry": set(),+ "pa-ai": {"pa-types"},+ "pa-models": {"pa-ai"},+ "pa-agent": {"pa-ai", "pa-types"},+ "pa-core": {"pa-agent", "pa-ai", "pa-models", "pa-types", "pa-telemetry"},+ "pa-daemon": {"pa-core"},+ "pa-tui": {"pa-types", "pa-core"},+ "pa-cli": {"pa-types", "pa-telemetry", "pa-ai", "pa-models",+ "pa-agent", "pa-core", "pa-daemon", "pa-tui"},+}+++def read_rows(path: Path) -> list[tuple[str, str]]:+ """The files-API rows, read raw: a leading-space path (` crates/...`)+ must stay the non-crate path it is, never be stripped into a crate+ mapping — the fields are tab-separated TSV, @tsv-escaped by jq, so no+ strip is needed or safe."""+ rows: list[tuple[str, str]] = []+ for line in path.read_text(encoding="utf-8").splitlines():+ if not line.strip():+ continue+ name, _, previous = line.partition("\t")+ rows.append((name, previous))+ return rows+++def select_crates(rows: list[tuple[str, str]]) -> list[str] | None:+ """The touched crates and reverse dependencies, or None for a full wave.++ Only known `crates/<pkg>/**` narrows. Everything else — the workflows,+ CI tooling, vendor/, the runtime sidecar, workspace-level Cargo files,+ install scripts, docs — affects more than any crate subset, and a rename+ counts both sides. An undecidable list (empty rows, the API ceiling or+ an unknown crate) fails safe to the full selection.+ """+ if not rows:+ return None+ if len(rows) >= FILES_API_CEILING:+ print(f"the PR has {len(rows)} changed files (the files API ceiling): "+ "the full selection runs")+ return None+ crates: set[str] = set()+ for name, previous in rows:+ if not name:+ # An empty filename column means the input is not the files+ # API's shape (a field-name regression): never select from it.+ print("an empty filename in the change list: the full selection runs")+ return None+ for candidate in (name, previous):+ if not candidate:+ continue # an ordinary (non-rename) row has no removed side+ parts = candidate.split("/")+ if len(parts) >= 2 and parts[0] == "crates" and parts[1]:+ if parts[1] not in WORKSPACE_DEPS:+ print(f"unknown crate {parts[1]}: the full selection runs")+ return None+ crates.add(parts[1])+ else:+ print(f"non-crate path {candidate}: the full selection runs")+ return None+ # pa-types owns the shared wire vocabulary: run every workspace test for+ # its changes, including the independent pa-telemetry leaf crate.+ if "pa-types" in crates:+ return sorted(WORKSPACE_DEPS)+ # Walk upward through the DAG until no dependent is left unselected.+ while True:+ dependents = {crate for crate, deps in WORKSPACE_DEPS.items()+ if deps & crates}+ expanded = crates | dependents+ if expanded == crates:+ return sorted(crates)+ crates = expanded+++def write_outputs(crates: list[str] | None, output_path: str) -> None:+ lines = []+ if crates is None:+ lines.append("crates=")+ lines.append("scope=all")+ else:+ lines.append("crates=" + ",".join(crates))+ lines.append(f"scope=crates ({len(crates)}: {', '.join(crates)})")+ if output_path and output_path != "/dev/null":+ with open(output_path, "a", encoding="utf-8") as handle:+ handle.write("\n".join(lines) + "\n")+ print("wave selection: " + lines[1].removeprefix("scope="))+ if crates:+ print("crates under test: " + ",".join(crates))+++def main() -> int:+ rows = read_rows(Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/dev/stdin"))+ crates = select_crates(rows)+ write_outputs(crates, os.environ.get("GITHUB_OUTPUT", "/dev/null"))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/ci_test_shard.py b/scripts/ci_test_shard.pyindex bac5018c0b..6193767452 100644--- a/scripts/ci_test_shard.py+++ b/scripts/ci_test_shard.py@@ -33,11 +33,23 @@ recorded in the shard manifest (JSON) the summary job audits: the union ofshard manifests must equal the full enumeration, so coverage can neversilently regress when targets are added or renamed.+The PR smoke gate narrows the selection, never the partition: `--crates+a,b` restricts the units this wave runs to the named packages' (a PR's+runtime smoke covers what it touched; the push wave runs the full union —+main is the authority). The stable crc32 assignment is untouched — a unit+still lands in exactly one shard of N — so flake attribution stays put and+the narrow and full runs of a unit are the same leg. Every shard records the+selection scope in its manifest, and the summary job audits the union+against the SELECTED set: coverage can never silently regress, and a+narrowed run can never masquerade as a full one.+Usage (from the repo root, e.g. in ci.yml):python3 scripts/ci_test_shard.py --shard 2 --total 4 \--manifest shard-manifest.jsonpython3 scripts/ci_test_shard.py --list # enumerate + shard assignment+ python3 scripts/ci_test_shard.py --shard 2 --total 8 --crates pa-tui \+ --print-selection # resolve + report the selection, run nothing"""from __future__ import annotations@@ -85,6 +97,10 @@ def shard_of(unit_id: str, total: int) -> int:return zlib.crc32(unit_id.encode("utf-8")) % total+def _unit_package(unit_id: str) -> str:+ return unit_id.split("#", 1)[0]++def enumerate_units() -> list[dict]:"""The workspace's test units, exactly what `cargo test --workspace` covers.@@ -177,15 +193,21 @@ def _failed_tests(lines: list[str]) -> list[str]:def write_manifest(path: Path, shard: int, total: int, all_ids: list[str],- results: list[dict]) -> None:+ results: list[dict], crates: list[str] | None = None) -> None:+ selection_ids = [i for i in all_ids+ if crates is None or _unit_package(i) in set(crates)]manifest = {"schema": 1,"shard": shard,"total": total,"all_unit_ids": all_ids,"digest": hashlib.sha256("\n".join(all_ids).encode("utf-8")).hexdigest(),+ "scope": scope_manifest(crates),+ "selected_unit_ids": selection_ids,"units": results,- "complete": len(results) == sum(1 for i in all_ids if shard_of(i, total) == shard - 1),+ # Complete means: every unit this wave SELECTED for this shard ran.+ "complete": len(results) == sum(1 for i in selection_ids+ if shard_of(i, total) == shard - 1),}path.parent.mkdir(parents=True, exist_ok=True)tmp = path.with_suffix(path.suffix + ".tmp")@@ -193,11 +215,14 @@ def write_manifest(path: Path, shard: int, total: int, all_ids: list[str],os.replace(tmp, path)-def summary_md(shard: int, total: int, results: list[dict]) -> str:+def summary_md(shard: int, total: int, results: list[dict],+ crates: list[str] | None = None) -> str:failed = [r for r in results if r["rc"] != 0]+ scope = ("full selection" if crates is None+ else f"crate selection: {', '.join(crates)}")lines = [f"### test shard {shard}/{total}",- f"- {len(results) - len(failed)}/{len(results)} units green",+ f"- {len(results) - len(failed)}/{len(results)} units green ({scope})",]if failed:lines.append(f"- **{len(failed)} failed:**")@@ -211,22 +236,81 @@ def summary_md(shard: int, total: int, results: list[dict]) -> str:return "\n".join(lines)+def parse_crates(raw: str | None) -> list[str] | None:+ """The selection scope: None = the full union; a crate list narrows it."""+ if raw is None or not raw.strip():+ return None+ crates = sorted({name.strip() for name in raw.split(",") if name.strip()})+ return crates or None+++def scope_manifest(crates: list[str] | None) -> dict:+ """The manifest's scope record (the summary audits against it)."""+ if crates is None:+ return {"kind": "all"}+ return {"kind": "crates", "crates": crates}+++def selected_units(units: list[dict], crates: list[str] | None) -> list[dict]:+ """The units this wave runs: the full union, or the named crates' subset."""+ if crates is None:+ return units+ return [u for u in units if u["package"] in set(crates)]++def main() -> int:parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])parser.add_argument("--shard", type=int, required=True, help="1-based shard index")parser.add_argument("--total", type=int, required=True, help="shard count")parser.add_argument("--manifest", type=Path, default=None,help="write the per-shard JSON manifest here")+ parser.add_argument("--crates", default=None,+ help="comma-separated crate names to run (the PR smoke's "+ "narrowed selection); omit for the full union")+ parser.add_argument("--print-selection", action="store_true",+ help="resolve + report this shard's selection, run nothing "+ "(the wave's conditional bins build reads it)")parser.add_argument("--list", action="store_true",help="print the enumeration and shard assignment, run nothing")args = parser.parse_args()if not 1 <= args.shard <= args.total:parser.error("--shard must be in 1..--total")+ crates = parse_crates(args.crates)+ if crates is not None and args.list:+ parser.error("--list enumerates the full workspace; it takes no --crates")units = enumerate_units()+ if crates is not None:+ # Fail closed on a mapping bug: a named package the workspace does+ # not build would narrow the wave to a silent zero (the selection+ # cannot name what the enumeration never saw).+ known = {u["package"] for u in units}+ unknown = sorted(set(crates) - known)+ if unknown:+ parser.error(+ f"--crates names packages the workspace does not build: "+ f"{', '.join(unknown)} (known: {', '.join(sorted(known))})")all_ids = [u["id"] for u in units]- mine = [u for u in units if shard_of(u["id"], args.total) == args.shard - 1]- print(f"[shard {args.shard}/{args.total}] {len(mine)}/{len(units)} units")+ selection = selected_units(units, crates)+ mine = [u for u in selection if shard_of(u["id"], args.total) == args.shard - 1]+ if crates is None:+ print(f"[shard {args.shard}/{args.total}] {len(mine)}/{len(units)} units "+ "(full selection)")+ else:+ print(f"[shard {args.shard}/{args.total}] {len(mine)}/{len(selection)} units "+ f"(crate selection: {', '.join(crates)}; the full union is {len(units)})")++ if args.print_selection:+ for u in mine:+ print(f" selected: {u['id']}")+ # The workflow's sibling-binary build reads this (an empty selection+ # needs no workspace bins): one machine line + the GITHUB_OUTPUT row.+ print(f"SELECTED={len(mine)}")+ github_output = os.environ.get("GITHUB_OUTPUT")+ if github_output:+ with open(github_output, "a", encoding="utf-8") as handle:+ handle.write(f"units={len(mine)}\n")+ return 0if args.list:for u in units:@@ -243,15 +327,17 @@ def main() -> int:f"in {result['seconds']}s")results.append(result)if args.manifest:- write_manifest(args.manifest, args.shard, args.total, all_ids, results)+ write_manifest(args.manifest, args.shard, args.total, all_ids,+ results, crates)if args.manifest:- write_manifest(args.manifest, args.shard, args.total, all_ids, results)+ write_manifest(args.manifest, args.shard, args.total, all_ids,+ results, crates)step_summary = os.environ.get("GITHUB_STEP_SUMMARY")if step_summary:Path(step_summary).parent.mkdir(parents=True, exist_ok=True)with open(step_summary, "a", encoding="utf-8") as f:- f.write(summary_md(args.shard, args.total, results) + "\n")+ f.write(summary_md(args.shard, args.total, results, crates) + "\n")failed = [r["id"] for r in results if r["rc"] != 0]if failed:diff --git a/scripts/ci_test_shard_summary.py b/scripts/ci_test_shard_summary.pyindex c1837d8d7a..2cb23bd274 100644--- a/scripts/ci_test_shard_summary.py+++ b/scripts/ci_test_shard_summary.py@@ -1,20 +1,26 @@#!/usr/bin/env python3"""Audit the shard manifests and merge their failure summaries (ci.yml).-The overall test gate is green only when EVERY shard reported and the union of-their executed units still covers the full enumeration. This script verifies-exactly that, so sharding can never silently drop coverage when test targets-are added, renamed, or move packages:+The overall test gate is green only when EVERY shard reported and the union+of their executed units still covers the run's SELECTION — the full+enumeration on push runs (main is the authority), or the touched crates'+subset on the PR smoke (`--crates`, recorded by every shard as the run's+`scope`). This script verifies exactly that, so sharding can never silently+drop coverage when test targets are added, renamed, or move packages — and a+narrowed run can never masquerade as a full one:1. every shard produced a manifest;2. all shards enumerated the same unit set (digest match);- 3. shard assignments are disjoint and their union is the whole set;- 4. every shard completed all of its assigned units;- 5. every executed unit passed.+ 3. all shards record the SAME selection scope (a mixed-scope wave is a+ broken partition; the summary names it instead of auditing a chimera);+ 4. shard assignments are disjoint and their union is the selected set+ (nothing selected is skipped, nothing outside the selection ran);+ 5. every shard completed all of its selected units;+ 6. every executed unit passed.-It then prints ONE merged report: which binaries failed, in which shard, with-their failing test names — the single place a lane looks when a PR run goes-red, instead of crawling four job logs.+It then prints ONE merged report: the scope, which binaries failed, in which+shard, with their failing test names — the single place a lane looks when a+PR run goes red, instead of crawling the job logs.Usage (from the repo root, in ci.yml's test summary job):@@ -44,6 +50,32 @@ def load_manifests(manifest_dir: Path, total: int):return manifests+def run_scope(manifests: dict) -> tuple[str, list[str] | None]:+ """The wave's selection scope, from the manifests (one source of truth)."""+ scopes = {json.dumps(m.get("scope", {"kind": "all"}), sort_keys=True)+ for m in manifests.values()}+ if len(scopes) != 1:+ return "MIXED", None+ scope = json.loads(next(iter(scopes)))+ return scope.get("kind", "all"), scope.get("crates")+++def selected_ids(manifest: dict, crates: list[str] | None) -> list[str]:+ """The selected universe: the manifest's recorded selection, or (older+ manifests without one) the full enumeration."""+ if "selected_unit_ids" in manifest:+ return manifest["selected_unit_ids"]+ all_ids = manifest["all_unit_ids"]+ if crates is None:+ return all_ids+ packages = set(crates)+ return [i for i in all_ids if _unit_package(i) in packages]+++def _unit_package(unit_id: str) -> str:+ return unit_id.split("#", 1)[0]++def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:"""Structural audit failures and the set of failing unit ids."""problems = []@@ -63,6 +95,17 @@ def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:return problems, failed_unitsall_ids = manifests[1]["all_unit_ids"]+ kind, crates = run_scope(manifests)+ if kind == "MIXED":+ described = sorted(json.dumps(m.get("scope"), sort_keys=True)+ for m in manifests.values())+ problems.append("shards recorded different selection scopes — a "+ "mixed-scope wave is a broken partition: " ++ ", ".join(described))+ return problems, failed_units+ selection = selected_ids(manifests[1], crates)+ selection_set = set(selection)+executed: dict[str, str] = {} # unit id -> shard that ran itfor shard, manifest in sorted(manifests.items()):for unit in manifest["units"]:@@ -78,16 +121,19 @@ def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:if unit["rc"] != 0:failed_units.add(uid)- missing = sorted(set(all_ids) - set(executed))+ missing = sorted(selection_set - set(executed))if missing:- problems.append(f"unassigned units (no shard ran them): {missing}")- extra = sorted(set(executed) - set(all_ids))+ problems.append(f"selected units no shard ran: {missing}")+ extra = sorted(set(executed) - selection_set)if extra:- problems.append(f"assigned outside the enumeration: {extra}")+ problems.append(f"units outside the selection ran: {extra}")+ outside = sorted(set(executed) - set(all_ids))+ if outside:+ problems.append(f"assigned outside the enumeration: {outside}")for shard, manifest in sorted(manifests.items()):if not manifest.get("complete", False):- assigned = [i for i in all_ids if shard_of(i, total) == shard - 1]+ assigned = [i for i in selection if shard_of(i, total) == shard - 1]unfinished = sorted(set(assigned) - {u["id"] for u in manifest["units"]})problems.append(f"shard {shard} is incomplete; units it never "f"reported: {unfinished}")@@ -96,7 +142,13 @@ def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:def merged_report(manifests: dict, total: int, problems: list[str],failed_units: set[str]) -> str:- lines = [f"### test summary ({total} shards)"]+ kind, crates = run_scope(manifests)+ scope = ("full selection" if kind == "all"+ else f"crate selection: {', '.join(crates or [])}")+ selected = selected_ids(manifests.get(1, {"all_unit_ids": []}), crates)+ union_size = len(manifests.get(1, {}).get("all_unit_ids", []))+ lines = [f"### test summary ({total} shards)",+ f"- scope: {scope} — {len(selected)} of {union_size} units selected"]for shard in sorted(manifests):manifest = manifests[shard]units = manifest["units"]diff --git a/scripts/release/restamp_continuous.py b/scripts/release/restamp_continuous.pynew file mode 100644index 0000000000..c43f8c5c64--- /dev/null+++ b/scripts/release/restamp_continuous.py@@ -0,0 +1,307 @@+#!/usr/bin/env python3+"""Restamp the continuous build's artifacts into a release-channel set.++The beta (nightly) release consumes the `continuous` workflow's artifacts+instead of rebuilding the 4-target matrix: `continuous` already built the+exact parent commit the nightly tag names (its own gates — GLIBC baseline,+split-debug, payload guards — ran green there, and nightly.yml waits out+that run before cutting the tag). The only differences between a continuous+artifact set and a release one are version-shaped, so this script restamps+them without touching the built bytes:++ - the tarball's inner `package.json` (the runtime version manifest the+ shipped binary reports via `--version`; see pa-cli config::version)+ moves from `<version>-continuous.<sha>` to the release version, keeping+ the `commit` provenance;+ - the archive, its SHA256SUMS line, and the (Linux) split-debug decoder+ rename to the release version;+ - the per-target manifest.json moves to the release version (per-binary+ and decoder rows kept in sync; `executableSha256`/`buildId` unchanged —+ the binaries are the continuous build's).++Everything else is verified, never rebuilt: the incoming archive must match+its own SHA256SUMS + manifest (an intact download), carry exactly the+continuous payload (assemble's designed entries plus `package.json`), be+stamped with the expected commit, and keep its decoder matching the+shipped ELF. Deterministic repacking reuses assemble_artifacts's own packer,+so the restamped archive is byte-equivalent to one assembled at the release+version from the same tree.++Usage (release.yml's consume job):++ python3 scripts/release/restamp_continuous.py \+ --incoming <dir with artifacts-<target>/ subdirs> \+ --version <x.y.z[-beta.N]> \+ --expect-commit <40-hex sha> \+ --out-dir <dir>+"""++from __future__ import annotations++import argparse+import hashlib+import json+import os+import platform+import re+import shutil+import subprocess+import sys+import tarfile+import tempfile+from pathlib import Path++# Same-directory release scripts (the packer + gates are reused, never re-implemented).+from assemble_artifacts import (+ TARGET_ALIASES,+ fail_if_decoder_in_archive,+ pack_tarball,+ sha256_file,+)+from bundle_catalog import validate_bundled_catalog_dir+from verify_release import CONTINUOUS_EXTRA_TOP_LEVEL, CONTINUOUS_SUFFIX, EXPECTED_TOP_LEVEL++# The runner-host target triple -> the binary this process can execute+# (installer.rs `target_for` parity: the same four continuous targets).+HOST_TARGETS = {+ ("linux", "x86_64"): "x86_64-unknown-linux-gnu",+ ("linux", "aarch64"): "aarch64-unknown-linux-gnu",+ ("macos", "aarch64"): "aarch64-apple-darwin",+ ("macos", "x86_64"): "x86_64-apple-darwin",+}+++def fail(message: str) -> None:+ print(f"error: {message}", file=sys.stderr)+ sys.exit(1)+++def host_target() -> str | None:+ return HOST_TARGETS.get((platform.system().lower(), platform.machine()))+++def read_sums(path: Path) -> dict[str, str]:+ """SHA256SUMS rows as {basename: digest}."""+ sums: dict[str, str] = {}+ for line in path.read_text(encoding="utf-8").splitlines():+ if not line.strip():+ continue+ digest, name = line.split(None, 1)+ sums[Path(name.strip()).name] = digest.strip()+ return sums+++def _member_target(archive: Path, staging: Path, member: tarfile.TarInfo) -> Path:+ """The extraction path for one member, proven inside `staging` first.++ A member name with an absolute path or a `..` component would write+ OUTSIDE the staging tree (the tarfile tar-slip class); the payload gate+ below runs only after the whole archive is read, so the traversal check+ belongs here, before any byte is written.+ """+ candidate = staging / member.name+ if member.name.startswith("/") or ".." in Path(member.name).parts:+ fail(f"archive {archive.name} carries an escaping member {member.name!r}; "+ "refusing to extract outside the staging tree")+ resolved = candidate.resolve()+ if staging.resolve() not in resolved.parents and resolved != staging.resolve():+ fail(f"archive {archive.name} carries an escaping member {member.name!r}; "+ "refusing to extract outside the staging tree")+ return candidate+++def unpack_archive(archive: Path, staging: Path) -> list[str]:+ """Extract a plain-file archive to `staging`; return its top-level entries."""+ with tarfile.open(archive, "r:gz") as tar:+ members = tar.getmembers()+ for member in members:+ if member.issym() or member.islnk():+ fail(f"archive {archive.name} contains a link entry {member.name!r}; "+ "refusing to restamp a payload the packer never produces")+ target = _member_target(archive, staging, member)+ if member.isdir():+ target.mkdir(parents=True, exist_ok=True)+ else:+ target.parent.mkdir(parents=True, exist_ok=True)+ source = tar.extractfile(member)+ assert source is not None+ with source, open(target, "wb") as sink:+ shutil.copyfileobj(source, sink)+ target.chmod(member.mode)+ return sorted({member.name.split("/", maxsplit=1)[0] for member in members})+++def glibc_baseline_ok(binary: Path) -> bool:+ """No symbol above GLIBC_2.35 (the Ubuntu 22.04 release baseline).++ The continuous build gated this inside its build container; the restamp+ re-proves it on the host it can execute.+ """+ result = subprocess.run(["objdump", "-T", str(binary)],+ capture_output=True, text=True, check=False)+ if result.returncode != 0:+ return False+ versions = [tuple(int(part) for part in symbol.split("_", 1)[1].split("."))+ for symbol in re.findall(r"GLIBC_[0-9.]+", result.stdout)]+ return bool(versions) and max(versions) <= (2, 35)+++def restamp_target(incoming: Path, out_dir: Path, version: str,+ expect_commit: str) -> dict:+ manifest = json.loads((incoming / "manifest.json").read_text(encoding="utf-8"))+ binaries = manifest["binaries"]+ if len(binaries) != 1:+ fail(f"{incoming.name}: expected exactly one binary row, found {len(binaries)}")+ binary = binaries[0]+ target = binary["target"]+ alias = TARGET_ALIASES[target]+ staged_commit = str(manifest.get("commit", "")).lower()+ if staged_commit != expect_commit:+ fail(f"{incoming.name}: artifact commit {staged_commit!r} != the tag's "+ f"parent {expect_commit!r}: these are not the continuous builds "+ "of the commit this release names")+ old_version = manifest["version"].removeprefix("v")+ old_archive_name = binary["file"]++ # 1. The download is intact: its own SHA256SUMS + manifest must agree.+ archive = incoming / old_archive_name+ if not archive.is_file():+ fail(f"{incoming.name}: archive {old_archive_name} missing")+ archive_sha = sha256_file(archive)+ sums = read_sums(incoming / "SHA256SUMS")+ if sums.get(old_archive_name) != archive_sha or binary["sha256"] != archive_sha:+ fail(f"{incoming.name}: {old_archive_name} does not match its "+ "SHA256SUMS/manifest digest — the download is not intact")++ # 2. The decoder rows (Linux) must be present and match their files.+ decoders = manifest.get("decoders", [])+ decoder_files = sorted(incoming.glob("*.debug.gz"))+ if target.endswith("-unknown-linux-gnu"):+ if len(decoders) != 1 or len(decoder_files) != 1:+ fail(f"{incoming.name}: a Linux target requires exactly one decoder "+ f"(manifest rows {len(decoders)}, files {len(decoder_files)})")+ if sha256_file(decoder_files[0]) != decoders[0]["sha256"]:+ fail(f"{incoming.name}: the decoder does not match its manifest sha256")+ if sums.get(decoder_files[0].name) != decoders[0]["sha256"]:+ fail(f"{incoming.name}: the decoder's SHA256SUMS line does not match")+ elif decoders or decoder_files:+ fail(f"{incoming.name}: unexpected decoder artifacts on {target}")++ # 3. Unpack; assert the continuous payload shape.+ staging = Path(tempfile.mkdtemp(prefix="prime-agent-restamp-"))+ try:+ entries = unpack_archive(archive, staging)+ expected = sorted(EXPECTED_TOP_LEVEL | CONTINUOUS_EXTRA_TOP_LEVEL)+ if entries != expected:+ fail(f"{incoming.name}: payload {entries} != the continuous "+ f"payload {expected}; refusing to restamp a foreign archive")+ package = json.loads((staging / "package.json").read_text(encoding="utf-8"))+ stamped = f"{old_version}-{CONTINUOUS_SUFFIX}.{expect_commit}"+ if package.get("version") != stamped:+ fail(f"{incoming.name}: package.json version "+ f"{package.get('version')!r} != the continuous stamp "+ f"{stamped!r}: not a continuous build's artifact")+ if sha256_file(staging / "prime-agent") != binary["executableSha256"]:+ fail(f"{incoming.name}: the staged binary does not match the "+ "manifest's executableSha256")+ validate_bundled_catalog_dir(staging)++ # 4. The runtime version manifest moves to the release version; the+ # built bytes stay the continuous build's.+ package["version"] = version+ (staging / "package.json").write_text(json.dumps(package, indent=2) + "\n")++ # 5. Repack deterministically (the same packer that built the archive).+ new_archive_name = f"prime-agent-{version}-{alias}.tar.gz"+ target_dir = out_dir / incoming.name+ target_dir.mkdir(parents=True, exist_ok=True)+ new_archive = target_dir / new_archive_name+ pack_tarball(staging, new_archive, entries)+ fail_if_decoder_in_archive(new_archive)+ new_sha = sha256_file(new_archive)++ # 6. The decoder rides along unchanged (bytes), renamed to the release version.+ new_lines = [f"{new_sha} {new_archive_name}"]+ if target.endswith("-unknown-linux-gnu"):+ new_decoder_name = f"prime-agent-{version}-{alias}.debug.gz"+ shutil.copyfile(decoder_files[0], target_dir / new_decoder_name)+ new_lines.append(f"{decoders[0]['sha256']} {new_decoder_name}")+ decoders[0]["file"] = new_decoder_name+ (target_dir / "SHA256SUMS").write_text("\n".join(new_lines) + "\n")++ # 7. The per-target manifest moves to the release version, keeping+ # the commit provenance of the build it consumed.+ binary["version"] = f"v{version}"+ binary["file"] = new_archive_name+ binary["sha256"] = new_sha+ manifest["version"] = f"v{version}"+ (target_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")++ # 8. The live gates, on the target this host can execute: the shipped+ # binary's `--version` must report the release version (the runtime+ # version manifest is what the update flow reads), and a GNU target+ # re-proves the GLIBC baseline. The other targets' binaries cannot+ # execute here; verify_release.py livechecked each on its native+ # continuous runner (the manifest's `commit` row names that build).+ livecheck = None+ if target == host_target():+ env = {k: v for k, v in os.environ.items() if k != "PI_PACKAGE_DIR"}+ run = subprocess.run([str(staging / "prime-agent"), "--version"],+ capture_output=True, text=True, env=env,+ cwd=staging, check=False)+ detail = (run.stdout or run.stderr).strip() or f"rc={run.returncode}"+ if run.returncode != 0 or run.stdout.strip() != version:+ fail(f"{incoming.name}: the restamped binary reports {detail!r}, "+ f"expected {version!r} — the runtime version manifest did not take")+ if target.endswith("-unknown-linux-gnu") \+ and not glibc_baseline_ok(staging / "prime-agent"):+ fail(f"{incoming.name}: the binary requires symbols above "+ "GLIBC_2.35 — above the Ubuntu 22.04 release baseline")+ livecheck = version+ return {+ "target": target,+ "platform": alias,+ "file": new_archive_name,+ "sha256": new_sha,+ "commit": expect_commit,+ "executableSha256": binary["executableSha256"],+ "livecheck": livecheck,+ }+ finally:+ shutil.rmtree(staging, ignore_errors=True)+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--incoming", required=True, type=Path,+ help="directory holding the downloaded artifacts-<target> sets")+ parser.add_argument("--version", required=True,+ help="the release version to stamp (e.g. 0.9.8-beta.1)")+ parser.add_argument("--expect-commit", required=True,+ help="the 40-hex commit the artifacts must be stamped with")+ parser.add_argument("--out-dir", required=True, type=Path,+ help="directory to write the restamped artifacts-<target> sets to")+ args = parser.parse_args()+ if not re.fullmatch(r"[0-9a-f]{40}", args.expect_commit):+ fail(f"invalid commit {args.expect_commit!r} (expected 40 hex chars)")+ if not re.fullmatch(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?", args.version):+ fail(f"invalid version {args.version!r}")++ incoming = [p for p in sorted(args.incoming.iterdir()) if p.is_dir()]+ if not incoming:+ fail(f"no artifacts-* directories under {args.incoming}")+ results = [restamp_target(directory, args.out_dir, args.version,+ args.expect_commit.lower())+ for directory in incoming]+ for result in results:+ print(json.dumps(result, indent=2))+ livechecked = [r["target"] for r in results if r["livecheck"]]+ print(f"restamped {len(results)} target sets to {args.version} "+ f"(continuous commit {args.expect_commit}; "+ f"livechecked: {', '.join(livechecked) or 'none on this host'}); "+ "the built bytes are unchanged")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/release/test_restamp.py b/scripts/release/test_restamp.pynew file mode 100644index 0000000000..d244760390--- /dev/null+++ b/scripts/release/test_restamp.py@@ -0,0 +1,370 @@+#!/usr/bin/env python3+"""Test battery for the continuous-artifact restamp (the consume route's+packaging half, release.yml's beta channel).++The beta release consumes the continuous build's artifacts and restamps+them to the beta version (scripts/release/restamp_continuous.py): no+rebuild, the built bytes stay the continuous build's. This battery pins the+restamp's contracts on a SYNTHETIC continuous artifact set built by the+real assembler (`assemble_artifacts.py --sha`, the exact shape continuous.yml+uploads), with the security case first: the restamp must refuse anything+that is not the named commit's continuous build.++The fixture binary mirrors the shipped runtime contract the restamp+relies on (pa-cli config::version): it prints the version from the+exe-adjacent `package.json` at runtime, falling back to a compiled-in+version when the manifest is unreadable — so the livecheck proves the+version restamp through the same mechanism the update flow reads.++Run: python3 scripts/release/test_restamp.py (or: make restamp-gates)+"""++from __future__ import annotations++import hashlib+import json+import shutil+import subprocess+import sys+import tarfile+import tempfile+import unittest+from pathlib import Path++SCRIPTS_DIR = Path(__file__).resolve().parent+REPO = SCRIPTS_DIR.parent.parent+sys.path.insert(0, str(SCRIPTS_DIR))++from assemble_artifacts import TARGET_ALIASES # noqa: E402 (same release-scripts directory)++ASSEMBLER = SCRIPTS_DIR / "assemble_artifacts.py"+SPLITTER = SCRIPTS_DIR / "split_debug.py"+BUNDLER = SCRIPTS_DIR / "bundle_catalog.py"+RESTAMPER = SCRIPTS_DIR / "restamp_continuous.py"+DECODER_VERIFIER = SCRIPTS_DIR / "verify_decoders.py"+VERIFIER = SCRIPTS_DIR / "verify_release.py"++HOST_TARGET = "x86_64-unknown-linux-gnu"+HOST_ALIAS = TARGET_ALIASES[HOST_TARGET]+DARWIN_TARGET = "x86_64-apple-darwin"+DARWIN_ALIAS = TARGET_ALIASES[DARWIN_TARGET]++# The runtime version resolver the shipped binary implements+# (pa-cli config::version parity: the exe-adjacent package.json wins at+# runtime; the compiled-in version is the fallback when the manifest is+# missing or malformed).+VERSION_RESOLVER_C = r'''#include <stdio.h>+#include <string.h>++int main(int argc, char **argv) {+ if (argc > 1 && strcmp(argv[1], "--version") == 0) {+ FILE *manifest = fopen("package.json", "r");+ if (manifest) {+ char line[512];+ while (fgets(line, sizeof(line), manifest)) {+ char *key = strstr(line, "\"version\"");+ if (!key) continue;+ char *start = strchr(key + 10, '"');+ if (!start) break;+ char *end = strchr(start + 1, '"');+ if (!end) break;+ fwrite(start + 1, 1, end - start - 1, stdout);+ putchar('\n');+ fclose(manifest);+ return 0;+ }+ fclose(manifest);+ }+ puts("%s");+ return 0;+ }+ puts("prime-agent (test fixture)");+ return 0;+}+'''+++def run_cli(script, args, cwd=None):+ return subprocess.run([sys.executable, str(script), *args],+ capture_output=True, text=True, cwd=cwd or REPO,+ check=False)+++def digest(path: Path) -> str:+ return hashlib.sha256(path.read_bytes()).hexdigest()+++class ContinuousFixture:+ """A synthetic continuous artifact set, the exact shape continuous.yml+ uploads: a real (tiny) ELF with a split-debug decoder, assembled by the+ real assembler with the commit stamp."""++ def __init__(self, root: Path, version: str, sha: str):+ self.root = root+ self.version = version+ self.sha = sha+ (root / "skills").mkdir(parents=True)+ (root / "skills" / "skill.md").write_text("# skill\n")+ (root / "LICENSE").write_text("license\n")+ (root / "README.md").write_text("readme\n")+ runtime = root / "runtime"+ runtime.mkdir()+ (runtime / "pyproject.toml").write_text("[project]\nname='rlm'\n")+ (runtime / "src").mkdir()+ (root / "prime-agent-runtime").mkdir()+ (root / "prime-agent-runtime" / "pyproject.toml").write_text("[project]\nname='rlm'\n")++ source = root / "resolver.c"+ source.write_text(VERSION_RESOLVER_C % version)+ self.raw = root / "bin" / "cargo-prime-agent"+ self.raw.parent.mkdir()+ subprocess.run(["gcc", "-g", "-Wl,--build-id", "-o", str(self.raw),+ str(source)], check=True)+ self.binary = root / "bin" / "prime-agent"+ self.decoder = root / "bin" / f"prime-agent-{version}-{HOST_ALIAS}.debug.gz"+ split = run_cli(SPLITTER, ["--binary", str(self.raw), "--shipped",+ str(self.binary), "--out", str(self.binary.parent),+ "--version", version, "--target", HOST_TARGET])+ if split.returncode:+ raise AssertionError(split.stderr)+ self.catalog = root / "catalog-assets"+ bundled = run_cli(BUNDLER, ["generate", "--fixture", "--out", str(self.catalog)])+ if bundled.returncode:+ raise AssertionError(bundled.stderr)+ self.dist = root / "dist"+ self.dist.mkdir()+ self.assemble_into(self.dist, sha=sha)++ def assemble_into(self, out_dir: Path, sha: str | None, target: str = HOST_TARGET):+ args = ["--repo-root", str(self.root), "--version", self.version,+ "--target", target, "--binary", str(self.binary),+ "--runtime-dir", str(self.root / "runtime"),+ "--out-dir", str(out_dir)]+ if target.endswith("-unknown-linux-gnu"):+ args += ["--decoder", str(self.decoder)]+ if sha:+ args += ["--sha", sha]+ args += ["--catalog-assets", str(self.catalog)]+ result = run_cli(ASSEMBLER, args)+ if result.returncode:+ raise AssertionError(result.stderr)+ # The CI build jobs point --decoder at the dist tree (split_debug+ # writes it there), so the decoder rides the assembled set; the+ # fixture mirrors that layout.+ if target.endswith("-unknown-linux-gnu"):+ shutil.copyfile(self.decoder,+ out_dir / f"prime-agent-{self.version}-{HOST_ALIAS}.debug.gz")+ return out_dir++ def artifact_dir(self, out_dir: Path, target: str = HOST_TARGET) -> Path:+ """The per-target layout continuous.yml uploads (download-artifact+ places one directory per artifact when a run has more than one)."""+ dist = out_dir / f"artifacts-{target}"+ dist.mkdir()+ archive = out_dir / f"prime-agent-{self.version}-{TARGET_ALIASES[target]}.tar.gz"+ shutil.move(str(archive), str(dist / archive.name))+ for name in ("SHA256SUMS", "manifest.json"):+ shutil.move(str(out_dir / name), str(dist / name))+ if target.endswith("-unknown-linux-gnu"):+ decoder = out_dir / f"prime-agent-{self.version}-{HOST_ALIAS}.debug.gz"+ shutil.move(str(decoder), str(dist / decoder.name))+ return dist+++class RestampTestCase(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.root = Path(self.tmp.name)+ self.commit = "a" * 40+ self.fixture = ContinuousFixture(self.root / "repo", "9.9.9", self.commit)+ self.incoming = self.root / "incoming"+ shutil.copytree(self.fixture.artifact_dir(self.fixture.dist),+ self.incoming / f"artifacts-{HOST_TARGET}")++ def tearDown(self):+ self.tmp.cleanup()++ def restamp(self, version="9.9.9-beta.2", commit=None, expect=0):+ out = self.root / "restamped"+ result = run_cli(RESTAMPER, [+ "--incoming", str(self.incoming), "--version", version,+ "--expect-commit", commit or self.commit, "--out-dir", str(out)])+ self.assertEqual(result.returncode, expect,+ f"restamp rc={result.returncode}: {result.stderr}")+ return result, out+++class TheProvenanceCase(RestampTestCase):+ """The restamp consumes ONLY the named commit's continuous build."""++ def test_a_foreign_commit_is_refused(self):+ result, _ = self.restamp(commit="b" * 40, expect=1)+ self.assertIn("not the continuous builds of the commit", result.stderr)++ def test_a_release_build_without_the_continuous_stamp_is_refused(self):+ plain = self.root / "plain-dist"+ plain.mkdir()+ self.fixture.assemble_into(plain, sha=None) # a release-shaped set, no package.json+ shutil.rmtree(self.incoming / f"artifacts-{HOST_TARGET}")+ shutil.copytree(self.fixture.artifact_dir(plain), self.incoming / f"artifacts-{HOST_TARGET}")+ result, _ = self.restamp(expect=1)+ # The provenance gate fires first (a release-shaped set carries no+ # commit row); either refusal names the missing continuous stamp.+ self.assertTrue(+ "not the continuous builds of the commit" in result.stderr+ or "not a continuous build's artifact" in result.stderr,+ result.stderr)++ def test_a_tampered_archive_is_refused(self):+ archive = self.incoming / f"artifacts-{HOST_TARGET}" / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"+ data = bytearray(archive.read_bytes())+ data[-1] ^= 0xFF+ archive.write_bytes(bytes(data))+ result, _ = self.restamp(expect=1)+ self.assertIn("does not match its SHA256SUMS", result.stderr)+++class TheRestampCase(RestampTestCase):+ """The restamped set is the release shape at the beta version, built+ bytes unchanged."""++ def test_the_continuous_artifact_passes_its_native_livecheck(self):+ # The continuous matrix now runs this verifier on every native runner+ # before uploading: exercise its --sha path on the assembled set.+ verified = run_cli(VERIFIER, [+ "--dist-dir", str(self.incoming / f"artifacts-{HOST_TARGET}"),+ "--version", self.fixture.version, "--target", HOST_TARGET,+ "--sha", self.commit])+ self.assertEqual(verified.returncode, 0, verified.stderr)+ self.assertIn("livecheck all OK", verified.stdout)++ def restamped_assertions(self, out: Path, version: str):+ target_dir = out / f"artifacts-{HOST_TARGET}"+ archive = target_dir / f"prime-agent-{version}-{HOST_ALIAS}.tar.gz"+ self.assertTrue(archive.is_file(), "the restamped archive is renamed to the version")+ manifest = json.loads((target_dir / "manifest.json").read_text())+ self.assertEqual(manifest["version"], f"v{version}")+ self.assertEqual(manifest["commit"], self.commit)+ binary = manifest["binaries"][0]+ self.assertEqual(binary["file"], archive.name)+ self.assertEqual(binary["sha256"], digest(archive))+ self.assertEqual(binary["executableSha256"],+ json.loads((self.incoming / f"artifacts-{HOST_TARGET}" / "manifest.json")+ .read_text())["binaries"][0]["executableSha256"])+ sums = {line.split()[1]: line.split()[0]+ for line in (target_dir / "SHA256SUMS").read_text().splitlines() if line.strip()}+ decoder_name = f"prime-agent-{version}-{HOST_ALIAS}.debug.gz"+ self.assertEqual(set(sums), {archive.name, decoder_name})+ # The decoder's bytes are the continuous build's, only renamed.+ old_decoder = self.incoming / f"artifacts-{HOST_TARGET}" / f"prime-agent-9.9.9-{HOST_ALIAS}.debug.gz"+ self.assertEqual(digest(target_dir / decoder_name), digest(old_decoder))+ self.assertEqual(manifest["decoders"][0]["file"], decoder_name)+ # The inner runtime version manifest is what the update flow reads.+ with tarfile.open(archive) as tar:+ package = json.load(tar.extractfile("package.json"))+ self.assertEqual(package["version"], version)+ self.assertEqual(package["commit"], self.commit)+ return target_dir++ def test_the_beta_version_lands_end_to_end(self):+ version = "9.9.9-beta.2"+ result, out = self.restamp(version=version)+ self.restamped_assertions(out, version)+ # The shipped binary reports the beta version through the package.json+ # the restamp rewrote (config::version parity: the exe-adjacent+ # manifest wins at runtime), and the promote-side decoder gate passes+ # on the restamped set — the two integration seams of the consume route.+ self.assertIn(f'"livecheck": "{version}"', result.stdout)+ verify = run_cli(DECODER_VERIFIER, [str(out)])+ self.assertEqual(verify.returncode, 0, verify.stderr)+ self.assertIn("OK", verify.stdout)+ # The release verifier accepts the restamped shape with its exact+ # gate set: payload, checksums, manifest, catalog assets, livecheck.+ target_dir = out / f"artifacts-{HOST_TARGET}"+ verified = run_cli(VERIFIER, [+ "--dist-dir", str(target_dir), "--version", version,+ "--target", HOST_TARGET, "--expect-package-json", version])+ self.assertEqual(verified.returncode, 0, verified.stderr)++ def test_a_non_gnu_target_restamps_without_a_decoder(self):+ darwin = self.fixture.assemble_into(self.root / "darwin-dist", sha=self.commit,+ target=DARWIN_TARGET)+ shutil.copytree(self.fixture.artifact_dir(darwin, target=DARWIN_TARGET),+ self.incoming / f"artifacts-{DARWIN_TARGET}")+ result, out = self.restamp(version="9.9.9-beta.7")+ target_dir = out / f"artifacts-{DARWIN_TARGET}"+ archive = target_dir / f"prime-agent-9.9.9-beta.7-{DARWIN_ALIAS}.tar.gz"+ self.assertTrue(archive.is_file())+ manifest = json.loads((target_dir / "manifest.json").read_text())+ self.assertEqual(manifest["version"], "v9.9.9-beta.7")+ self.assertNotIn("decoders", manifest)+ sums = (target_dir / "SHA256SUMS").read_text().splitlines()+ self.assertEqual(len(sums), 1)+ # The foreign-arch set cannot execute on this host: restamp skips its+ # livecheck; continuous verified it earlier on its native runner.+ self.assertIn('"livecheck": null', result.stdout)++ def test_an_escaping_tar_member_is_refused_before_any_write(self):+ """The tar-slip class: a member named `..`/absolute must never be+ extracted outside the staging tree (the payload gate runs only+ after the unpack, so the traversal guard is the unpack's own)."""+ import tarfile+ staging = self.root / "sneak"+ staging.mkdir()+ target = self.incoming / f"artifacts-{HOST_TARGET}"+ archive = target / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"+ with tarfile.open(archive) as tar:+ tar.extractall(staging)+ malicious = staging / "prime-agent-9.9.9-evil.tar.gz"+ with tarfile.open(malicious, "w:gz") as tar:+ info = tarfile.TarInfo("../escaped-payload")+ payload = b"outside the staging tree"+ info.size = len(payload)+ tar.addfile(info, __import__("io").BytesIO(payload))+ import hashlib+ new_sha = hashlib.sha256(malicious.read_bytes()).hexdigest()+ shutil.move(str(malicious), str(archive))+ sums = (target / "SHA256SUMS").read_text().splitlines()+ (target / "SHA256SUMS").write_text("\n".join(+ new_sha + line[64:] if line.split()[1] == archive.name else line+ for line in sums) + "\n")+ manifest = json.loads((target / "manifest.json").read_text())+ manifest["binaries"][0]["sha256"] = new_sha+ (target / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")+ result, _ = self.restamp(expect=1)+ self.assertIn("escaping member", result.stderr)+ self.assertFalse((self.root / "escaped-payload").exists(),+ "no byte may land outside the staging tree")++ def test_repacking_is_deterministic(self):+ _, first = self.restamp(version="9.9.9-beta.3")+ first_archive = digest(first / f"artifacts-{HOST_TARGET}" /+ f"prime-agent-9.9.9-beta.3-{HOST_ALIAS}.tar.gz")+ _, second = self.restamp(version="9.9.9-beta.3")+ second_archive = digest(second / f"artifacts-{HOST_TARGET}" /+ f"prime-agent-9.9.9-beta.3-{HOST_ALIAS}.tar.gz")+ self.assertEqual(first_archive, second_archive)++ def test_a_foreign_payload_is_refused(self):+ # Sneak an unexpected top-level file into the set and rebuild the+ # archive deterministically; the sums stay stale, so this also proves+ # the integrity gate fires before the payload gate ever matters.+ staging = self.root / "sneak"+ staging.mkdir()+ archive = self.incoming / f"artifacts-{HOST_TARGET}" / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"+ with tarfile.open(archive) as tar:+ tar.extractall(staging)+ (staging / "EXTRA").write_text("unexpected\n")+ entries = sorted(p.name for p in staging.iterdir())+ from assemble_artifacts import pack_tarball+ rebuilt = staging / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"+ pack_tarball(staging, rebuilt, entries)+ shutil.move(str(rebuilt), str(archive))+ result, _ = self.restamp(expect=1)+ self.assertTrue("does not match its SHA256SUMS" in result.stderr+ or "!= the continuous payload" in result.stderr,+ result.stderr)+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/scripts/release/verify_release.py b/scripts/release/verify_release.pyindex 8a1c8c0664..84f2cb20da 100644--- a/scripts/release/verify_release.py+++ b/scripts/release/verify_release.py@@ -82,12 +82,19 @@ def main() -> int:parser.add_argument("--target", required=True)parser.add_argument("--sha", default=None,help="commit SHA the tarball must be stamped with")+ parser.add_argument("--expect-package-json", default=None, metavar="VERSION",+ help="the consume-route restamp shape: a package.json "+ "carrying exactly this release version")args = parser.parse_args()if args.sha is not None:args.sha = args.sha.lower()+ if args.expect_package_json and args.sha:+ fail("--sha (the continuous stamp) and --expect-package-json "+ "(the restamped release shape) are mutually exclusive")expected_top_level = EXPECTED_TOP_LEVEL | (- CONTINUOUS_EXTRA_TOP_LEVEL if args.sha else set()+ CONTINUOUS_EXTRA_TOP_LEVEL if (args.sha or args.expect_package_json)+ else set())archive_name = (@@ -158,6 +165,14 @@ def main() -> int:# The bundled catalog assets must be present and valid in the# installed layout (the full packer gates: no small-fixture waiver).catalog_facts = validate_bundled_catalog_dir(scratch)+ if args.expect_package_json is not None:+ stamped = json.loads((scratch / "package.json").read_text(encoding="utf-8"))+ if stamped.get("version") != args.expect_package_json:+ fail(f"package.json version is {stamped.get('version')!r}, "+ f"expected the restamped release version "+ f"{args.expect_package_json!r}")+ if not str(stamped.get("commit", "")).strip():+ fail("the restamped package.json lost its commit provenance")binary = scratch / "prime-agent"if not os.access(binary, os.X_OK):fail("staged prime-agent is not executable")@@ -174,6 +189,8 @@ def main() -> int:expected_version = (f"{args.version}-{CONTINUOUS_SUFFIX}.{args.sha}" if args.sha else args.version)+ if args.expect_package_json is not None:+ expected_version = args.expect_package_jsonif version_out != expected_version:fail(f"staged prime-agent reports {version_out!r}, "diff --git a/scripts/test_ci_pr_crates.py b/scripts/test_ci_pr_crates.pynew file mode 100644index 0000000000..a9d490a3a3--- /dev/null+++ b/scripts/test_ci_pr_crates.py@@ -0,0 +1,161 @@+#!/usr/bin/env python3+"""Test battery for the PR-files-to-crates mapping (ci.yml's changes job).++The contracts, pinned on real pulls/files-API shapes (the `filename` ++`previous_filename` rows — the API's own field names, so a field-name+regression fails safe instead of silently narrowing to nothing):+ - a crate-only PR maps to those crates and their transitive dependents;+ - ANY non-crate path - and a rename's removed side too - fails safe to+ the full selection;+ - an undecidable list (empty input, an empty filename row, the API's+ 3000-file ceiling) fails safe to the full selection, never to an empty+ selection: a narrowed wave that runs nothing would masquerade as green.++Run: python3 scripts/test_ci_pr_crates.py (or: make shard-gates)+"""++from __future__ import annotations++import importlib.util+import os+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++SCRIPTS_DIR = Path(__file__).resolve().parent+sys.path.insert(0, str(SCRIPTS_DIR))++import ci_pr_crates # noqa: E402 (the tool under test)++MAPPER = SCRIPTS_DIR / "ci_pr_crates.py"+++def run_mapper(rows: list[str]) -> tuple[int, str, dict[str, str]]:+ output = Path(tempfile.mkdtemp()) / "github-output"+ env = dict(os.environ)+ env["GITHUB_OUTPUT"] = str(output)+ listing = Path(tempfile.mkdtemp()) / "pr-files.tsv"+ listing.write_text("\n".join(rows) + "\n", encoding="utf-8")+ result = subprocess.run([sys.executable, str(MAPPER), str(listing)],+ capture_output=True, text=True, env=env, check=False)+ outputs = {}+ for line in output.read_text().splitlines():+ key, _, value = line.partition("=")+ outputs[key] = value+ return result.returncode, result.stdout, outputs+++class TheMappingTestCase(unittest.TestCase):++ def test_a_crate_only_pr_narrows_to_those_crates(self):+ rc, out, outputs = run_mapper([+ "crates/pa-daemon/src/main.rs",+ "crates/pa-daemon/src/roster.rs",+ "crates/pa-tui/src/view/runs.rs",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "pa-cli,pa-daemon,pa-tui")+ self.assertIn("crates (3: pa-cli, pa-daemon, pa-tui)", outputs["scope"])++ def test_any_non_crate_path_fails_safe_to_the_full_selection(self):+ rc, out, outputs = run_mapper([+ "crates/pa-core/src/lib.rs",+ ".github/workflows/ci.yml",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertEqual(outputs["scope"], "all")+ self.assertIn("non-crate path .github/workflows/ci.yml", out)++ def test_a_renames_removed_side_counts_too(self):+ # A rename from a non-crate path INTO a crate: the PR removed the+ # old path, so the full selection runs.+ rc, out, outputs = run_mapper([+ "crates/pa-cli/src/new.rs\tdocs/old-location.md",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertIn("non-crate path docs/old-location.md", out)++ def test_a_rename_within_crates_narrows_to_both_crates(self):+ rc, out, outputs = run_mapper([+ "crates/pa-tui/src/editor/selection.rs\tcrates/pa-tui/src/selection.rs",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "pa-cli,pa-tui")++ def test_pa_types_only_runs_every_crate(self):+ rc, out, outputs = run_mapper(["crates/pa-types/src/lib.rs"])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], ",".join(sorted(ci_pr_crates.WORKSPACE_DEPS)))++ def test_pa_tui_only_runs_pa_cli_too(self):+ rc, out, outputs = run_mapper(["crates/pa-tui/src/lib.rs"])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "pa-cli,pa-tui")++ def test_pa_telemetry_only_runs_all_dependents(self):+ rc, out, outputs = run_mapper(["crates/pa-telemetry/src/lib.rs"])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"],+ "pa-cli,pa-core,pa-daemon,pa-telemetry,pa-tui")++ def test_an_unknown_crate_fails_safe(self):+ rc, out, outputs = run_mapper(["crates/pa-future/src/lib.rs"])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertIn("unknown crate pa-future", out)++ def test_an_empty_filename_row_fails_safe(self):+ # The field-name regression the reviewer demanded pinned: a row whose+ # filename column is empty (e.g. reading `.path` from the files API,+ # whose field is `filename`) must never narrow the wave to nothing -+ # the run fails safe to the full selection instead.+ rc, out, outputs = run_mapper([+ "\tcrates/pa-core/src/lib.rs",+ "crates/pa-daemon/src/lib.rs",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertEqual(outputs["scope"], "all")+ self.assertIn("empty filename", out)++ def test_a_null_field_read_also_fails_safe(self):+ # jq renders a missing field as the string "null": not a crate path,+ # so the full selection runs (nothing narrows on a broken column).+ rc, out, outputs = run_mapper([+ "null\tdocs/removed.md",+ "crates/pa-core/src/lib.rs",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")++ def test_an_empty_list_fails_safe_to_the_full_selection(self):+ rc, out, outputs = run_mapper([])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertEqual(outputs["scope"], "all")++ def test_a_whitespace_prefixed_path_is_a_non_crate_path(self):+ # The fields are read raw: a path with a leading space (or any+ # non-crate prefix) must never be stripped into a crate mapping.+ rc, out, outputs = run_mapper([+ " crates/pa-core/src/lib.rs",+ "crates/pa-tui/src/lib.rs",+ ])+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertIn("non-crate path crates/pa-core/src/lib.rs", out)++ def test_the_files_api_ceiling_fails_safe(self):+ rows = ["crates/pa-core/src/lib.rs"] * ci_pr_crates.FILES_API_CEILING+ rc, out, outputs = run_mapper(rows)+ self.assertEqual(rc, 0, out)+ self.assertEqual(outputs["crates"], "")+ self.assertIn("ceiling", out)+++if __name__ == "__main__":+ unittest.main(verbosity=2)diff --git a/scripts/test_ci_test_shard.py b/scripts/test_ci_test_shard.pynew file mode 100644index 0000000000..e1fe8aa1cc--- /dev/null+++ b/scripts/test_ci_test_shard.py@@ -0,0 +1,251 @@+#!/usr/bin/env python3+"""Test battery for the CI shard tooling (scripts/ci_test_shard.py ++scripts/ci_test_shard_summary.py): the PR smoke's narrowed selection and+the summary's scope-aware partition audit.++The contracts, pinned on the REAL workspace enumeration (cargo metadata+only - no builds):+ - the stable crc32 assignment is untouched by --crates: a narrowed run's+ unit lands in the same shard the full run assigns it;+ - the full selection is the default (push waves) and audits green;+ - a crate selection narrows the universe; the union of the shards' runs+ must cover EXACTLY the selection - a narrowed run can never+ masquerade as a full one, and coverage can never silently regress;+ - a mixed-scope wave (one shard claiming a different selection) fails+ the audit instead of auditing a chimera;+ - a shard that skips one of its selected units fails;+ - a unit that runs outside the selection fails;+ - --print-selection reports the selection without running anything and+ emits the units count the wave's conditional bins build reads.++Run: python3 scripts/test_ci_test_shard.py (or: make shard-gates)+"""++from __future__ import annotations++import importlib.util+import json+import os+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++SCRIPTS_DIR = Path(__file__).resolve().parent+REPO = SCRIPTS_DIR.parent++TOTAL = 8+++def load_module(name: str, path: Path):+ spec = importlib.util.spec_from_file_location(name, path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++SHARD = load_module("ci_test_shard", SCRIPTS_DIR / "ci_test_shard.py")+SUMMARY = load_module("ci_test_shard_summary", SCRIPTS_DIR / "ci_test_shard_summary.py")+++def run_summary(directory: Path):+ return subprocess.run(+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard_summary.py"),+ "--total", str(TOTAL), "--dir", str(directory)],+ capture_output=True, text=True, check=False)+++class EnumerationTestCase(unittest.TestCase):+ """The stable assignment + the narrowed selection."""++ @classmethod+ def setUpClass(cls):+ cls.units = SHARD.enumerate_units()+ cls.all_ids = [u["id"] for u in cls.units]+ cls.packages = {u["package"] for u in cls.units}++ def write_wave(self, directory: Path, crates, results_by_shard=None,+ scope_by_shard=None):+ selection = SHARD.selected_units(self.units, crates)+ for shard in range(1, TOTAL + 1):+ mine = [u for u in selection+ if SHARD.shard_of(u["id"], TOTAL) == shard - 1]+ results = results_by_shard(shard, mine) if results_by_shard \+ else [dict(id=u["id"], package=u["package"], kind=u["kind"],+ name=u["name"], rc=0, seconds=1.0,+ failed_tests=[]) for u in mine]+ SHARD.write_manifest(+ directory / f"shard-manifest-{shard}.json", shard, TOTAL,+ self.all_ids, results, scope_by_shard(shard) if scope_by_shard else crates)+ return selection++ def test_the_assignment_is_stable_under_the_selection(self):+ """The narrowed universe never moves a unit between shards: crc32+ mod 8 is computed on the unit id, not the selection."""+ for unit in self.units:+ shard = SHARD.shard_of(unit["id"], TOTAL)+ self.assertEqual(unit["id"],+ [u["id"] for u in self.units+ if SHARD.shard_of(u["id"], TOTAL) == shard+ and u["id"] == unit["id"]][0])++ def test_crate_selection_covers_exactly_that_crates_units(self):+ crate = sorted(p for p in self.packages if p != "pa-agent")[0]+ selection = SHARD.selected_units(self.units, [crate])+ self.assertTrue(selection, "the crate selection is never empty")+ self.assertTrue(all(u["package"] == crate for u in selection))+ full = SHARD.selected_units(self.units, None)+ self.assertLess(len(selection), len(full))+ self.assertTrue(set(u["id"] for u in selection) <= set(u["id"] for u in full))+++class AuditTestCase(unittest.TestCase):+ """The summary's scope-aware partition audit."""++ @classmethod+ def setUpClass(cls):+ cls.units = SHARD.enumerate_units()+ cls.all_ids = [u["id"] for u in cls.units]+ # The audit-failure cases need a crate whose units spread across+ # several shards: the largest unit count guarantees it.+ counts = {}+ for unit in cls.units:+ counts[unit["package"]] = counts.get(unit["package"], 0) + 1+ cls.crate = max((p for p in counts if p != "pa-agent"),+ key=lambda p: counts[p])+ cls.selection = [u["id"] for u in cls.units+ if u["package"] == cls.crate]++ def green_wave(self, directory: Path, crates, break_shard=None,+ drop_unit=False, add_outside=False, scope_by_shard=None):+ selection = [u["id"] for u in self.units+ if crates is None or u["package"] in set(crates)]+ for shard in range(1, TOTAL + 1):+ mine = [i for i in selection if SHARD.shard_of(i, TOTAL) == shard - 1]+ results = [dict(id=i, rc=0, seconds=1.0, failed_tests=[]) for i in mine]+ if break_shard == shard and drop_unit and results:+ results = results[:-1]+ if break_shard == shard and add_outside:+ outside = [i for i in self.all_ids+ if i not in set(selection)+ and SHARD.shard_of(i, TOTAL) == shard - 1]+ if outside:+ results.append(dict(id=outside[0], rc=0, seconds=1.0,+ failed_tests=[]))+ use_crates = scope_by_shard(shard) if scope_by_shard else crates+ SHARD.write_manifest(+ directory / f"shard-manifest-{shard}.json", shard, TOTAL,+ self.all_ids, results, use_crates)++ def test_a_full_wave_audits_green(self):+ with tempfile.TemporaryDirectory() as tmp:+ directory = Path(tmp)+ self.green_wave(directory, None)+ result = run_summary(directory)+ self.assertEqual(result.returncode, 0, result.stdout)+ self.assertIn("full selection", result.stdout)++ def test_a_narrowed_wave_audits_green_and_names_the_scope(self):+ with tempfile.TemporaryDirectory() as tmp:+ directory = Path(tmp)+ self.green_wave(directory, [self.crate])+ result = run_summary(directory)+ self.assertEqual(result.returncode, 0, result.stdout)+ self.assertIn(f"crate selection: {self.crate}", result.stdout)+ self.assertIn(f"{len(self.selection)} of {len(self.all_ids)} units selected",+ result.stdout)++ def test_a_mixed_scope_wave_fails_the_audit(self):+ with tempfile.TemporaryDirectory() as tmp:+ directory = Path(tmp)+ self.green_wave(directory, [self.crate],+ scope_by_shard=lambda shard: None if shard == 5 else [self.crate])+ result = run_summary(directory)+ self.assertEqual(result.returncode, 1, result.stdout)+ self.assertIn("different selection scopes", result.stdout)++ def test_a_shard_that_skips_a_selected_unit_fails(self):+ with tempfile.TemporaryDirectory() as tmp:+ directory = Path(tmp)+ self.green_wave(directory, [self.crate], break_shard=4, drop_unit=True)+ result = run_summary(directory)+ self.assertEqual(result.returncode, 1, result.stdout)+ self.assertIn("never reported", result.stdout)++ def test_a_unit_that_runs_outside_the_selection_fails(self):+ with tempfile.TemporaryDirectory() as tmp:+ directory = Path(tmp)+ self.green_wave(directory, [self.crate], break_shard=2, add_outside=True)+ result = run_summary(directory)+ self.assertEqual(result.returncode, 1, result.stdout)+ self.assertIn("outside the selection ran", result.stdout)++ def test_a_selected_unit_no_shard_ran_fails(self):+ with tempfile.TemporaryDirectory() as tmp:+ directory = Path(tmp)+ self.green_wave(directory, [self.crate])+ # Drop the unit from the run AND from the shard's recorded+ # selection reality: rewrite one manifest with the unit missing+ # from its units and its complete flag falsified.+ shard = next(SHARD.shard_of(i, TOTAL) + 1 for i in self.selection)+ path = directory / f"shard-manifest-{shard}.json"+ manifest = json.loads(path.read_text())+ if manifest["units"]:+ manifest["units"] = manifest["units"][:-1]+ manifest["complete"] = False+ path.write_text(json.dumps(manifest, indent=1) + "\n")+ result = run_summary(directory)+ self.assertEqual(result.returncode, 1, result.stdout)+++class SelectionReportingTestCase(unittest.TestCase):+ """--print-selection resolves without running and emits the count."""++ def test_print_selection_reports_and_counts(self):+ env = dict(os.environ)+ output = Path(tempfile.mkdtemp()) / "github-output"+ env["GITHUB_OUTPUT"] = str(output)+ result = subprocess.run(+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard.py"),+ "--shard", "3", "--total", str(TOTAL), "--crates", "pa-daemon",+ "--print-selection"],+ capture_output=True, text=True, env=env, cwd=REPO, check=False)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertIn("SELECTED=", result.stdout)+ self.assertIn("crate selection: pa-daemon", result.stdout)+ count = int(next(line for line in result.stdout.splitlines()+ if line.startswith("SELECTED=")).split("=")[1])+ self.assertGreater(count, 0)+ self.assertIn(f"units={count}", output.read_text())++ def test_an_empty_selection_reports_zero_and_skips_nothing(self):+ """A crate whose units all hash to OTHER shards leaves a leg with+ nothing to run: the bins build reads units=0 and skips."""+ env = dict(os.environ)+ output = Path(tempfile.mkdtemp()) / "github-output"+ env["GITHUB_OUTPUT"] = str(output)+ # pa-ai's two units both hash to shard 8: every other leg selects zero.+ result = subprocess.run(+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard.py"),+ "--shard", "3", "--total", str(TOTAL), "--crates", "pa-ai",+ "--print-selection"],+ capture_output=True, text=True, env=env, cwd=REPO, check=False)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertIn("SELECTED=0", result.stdout)+ self.assertIn("units=0", output.read_text())++ def test_an_unknown_package_is_refused_fail_closed(self):+ """A --crates name the workspace does not build is a mapping bug:+ refuse it (a silent zero would masquerade as a green wave)."""+ result = subprocess.run(+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard.py"),+ "--shard", "3", "--total", str(TOTAL), "--crates", "pa-nonexistent",+ "--print-selection"],+ capture_output=True, text=True, cwd=REPO, check=False)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("does not build", result.stderr)+++if __name__ == "__main__":+ unittest.main(verbosity=2)