Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

reuse build artifacts in the beta flow and filter tests by path (#3187)

kt776c911ecba4
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a82e3371b8..8f6d720c54 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,9 +17,29 @@
# #2627's fork_from test-target errors land as latent compile breaks, and
# #2635's login-recovery test land red — none noticed by CI.
#
-# pull_request against main/rust — the same gates on the merge candidate,
+# pull_request against main — the same gates on the merge candidate,
# so the merge bar stops relying solely on review bots.
#
+# merge_group — full workspace tests on the queued merge candidate,
+# not the PR's narrowed selection. Branch protection must separately
+# enable merge queue and require the final gates.
+#
+# THE PR SMOKE MODEL (the touched-crate path filter): the PR wave narrows
+# the TEST phase to touched crates and their reverse dependents' units
+# (the `changes` job maps the PR's changed files to crates; any path
+# outside crates/<pkg>/** fails safe
+# to the full selection — CI/scripts/vendor/runtime changes affect the
+# whole suite, and docs-only waste is cheaper than a silent hole). The
+# stable crc32 shard assignment is untouched — only the unit selection
+# narrows — and the push wave keeps running the full union, so main is the
+# authority: the PR proves the touched crates' runtime behavior fast, the
+# merge proves the whole workspace. fmt + check/clippy stay full-scope on
+# PRs (the compile authority); reverse-dep selection also runs tests of
+# crates affected by a dependency's behavior change. The shard manifests
+# record the selection scope and the summary job audits the union against it, so a
+# narrowed run can never masquerade as a full one and coverage can never
+# silently regress.
+#
# The test phase is sharded: `cargo test --workspace` ran every test binary
# serially in one job (~13 minutes; the per-PR wall-clock floor). It is now
# the union of SHARD_COUNT parallel shard jobs: scripts/ci_test_shard.py enumerates
@@ -49,15 +69,17 @@ on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
branches: [main]
+ merge_group:
+ types: [checks_requested]
permissions:
contents: read
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
- # PRs: a newer push supersedes the stale gates. Pushes: the newest tip
- # supersedes the older run — under this fleet's merge cadence a queued
- # (non-canceling) push run would lag hours behind and bill stale tips.
+ # PRs: a newer push supersedes stale gates. Queue entries use their own
+ # merge-group ref; a superseded entry's gates can be cancelled. Pushes:
+ # the newest tip supersedes the older run instead of billing stale tips.
cancel-in-progress: true
env:
@@ -82,13 +104,62 @@ env:
SHARD_COUNT: "8"
jobs:
+ # The touched-crate path filter (the PR smoke's selection). Push and queue
+ # runs use the full selection; PR runs select crates plus reverse deps,
+ # failing safe to the full selection on any non-crate path. The output
+ # feeds test shards' --crates; empty means "the full union".
+ changes:
+ name: touched crates (PR) / full selection (push + queue)
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft
+ outputs:
+ crates: ${{ steps.select.outputs.crates }}
+ scope: ${{ steps.select.outputs.scope }}
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - name: Select the touched crates
+ id: select
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ run: |
+ set -euo pipefail
+ if [ "${{ github.event_name }}" != "pull_request" ]; then
+ echo "${{ github.event_name }} wave: the full selection"
+ echo "crates=" >> "${GITHUB_OUTPUT}"
+ echo "scope=all" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ # The PR's exact change set: the pulls files API (merge-base
+ # semantics, renames carrying both sides), fully paginated —
+ # `gh pr view --json files` caps at 100 rows and would silently
+ # under-map a large PR, and its field names differ from the REST
+ # rows (`path` vs `filename`). The mapping itself is fail-safe
+ # by construction (scripts/ci_pr_crates.py + its battery): any
+ # non-crate path, a rename's removed side, an empty column, or
+ # the API's 3000-file ceiling means the FULL selection runs.
+ gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \
+ --paginate --jq '.[] | [.filename, .previous_filename // ""] | @tsv' \
+ > "${RUNNER_TEMP}/pr-files.tsv"
+ python3 scripts/ci_pr_crates.py "${RUNNER_TEMP}/pr-files.tsv"
+ - name: Report the selection
+ run: |
+ set -euo pipefail
+ echo "wave selection: ${{ steps.select.outputs.scope }}"
+ if [ -n "${{ steps.select.outputs.crates }}" ]; then
+ echo "crates under test: ${{ steps.select.outputs.crates }}"
+ fi
+
fmt:
name: fmt
runs-on: ubuntu-24.04
timeout-minutes: 30
# Draft PRs do not gate (no billing on work-in-progress pushes); the
# ready_for_review trigger runs the gates once a draft is marked ready.
- if: github.event_name == 'push' || !github.event.pull_request.draft
+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
@@ -103,7 +174,7 @@ jobs:
name: check + clippy (workspace)
runs-on: ubuntu-24.04
timeout-minutes: 45
- if: github.event_name == 'push' || !github.event.pull_request.draft
+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
@@ -135,12 +206,16 @@ jobs:
# ONLY its test subset. The union of the shards is exactly the old serial
# `cargo test --workspace`.
test-shard:
+ # The shard legs need the wave's selection (the changes job's crates
+ # output) before they can decide what to run — and whether the
+ # sibling-binary build is needed at all.
+ needs: [changes]
# strategy.job-total (the matrix leg count) auto-derives the displayed
# total; the env context is not reachable from a job name.
name: test shard ${{ matrix.shard }}/${{ strategy.job-total }}
runs-on: ubuntu-24.04
timeout-minutes: 45
- if: github.event_name == 'push' || !github.event.pull_request.draft
+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft
strategy:
# One red shard must not cancel the others: their own conclusions are
# the per-shard signal the summary merges.
@@ -168,18 +243,30 @@ jobs:
run: |
python3 -m pip install --user uv
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+ # The wave's selection, resolved once per leg before anything builds:
+ # the narrowed selection's unit list decides whether the bins build
+ # below has any work to do (an empty selection runs no units and needs
+ # no sibling binaries).
+ - name: Resolve this shard's selection
+ id: selection
+ run: |
+ set -euo pipefail
+ python3 scripts/ci_test_shard.py --shard ${{ matrix.shard }} --total ${{ env.SHARD_COUNT }} --crates "${{ needs.changes.outputs.crates }}" --print-selection
# Sibling-binary parity: the serial `cargo test --workspace` built every
# workspace bin into the shared profile dir as a side effect, and the
# e2e suites look each other's binaries up there (pa-cli's
# daemon_commands_e2e spawns pa-daemon this way). A shard builds its
# test subset only, so the bins are built explicitly once up front —
# first measured on run 36085993711, where the missing pa-daemon bin
- # failed exactly this suite.
+ # failed exactly this suite. A leg with no selected units skips this
+ # (and runs nothing below); the push wave always selects units, so its
+ # behavior is unchanged.
- name: Build the workspace bins (sibling-binary parity for the e2e suites)
+ if: steps.selection.outputs.units != '0'
run: cargo build --locked --workspace --bins
- name: Run this shard's test subset
run: |
- python3 scripts/ci_test_shard.py --shard ${{ matrix.shard }} --total ${{ env.SHARD_COUNT }} --manifest "$RUNNER_TEMP/shard-manifest-${{ matrix.shard }}.json"
+ python3 scripts/ci_test_shard.py --shard ${{ matrix.shard }} --total ${{ env.SHARD_COUNT }} --crates "${{ needs.changes.outputs.crates }}" --manifest "$RUNNER_TEMP/shard-manifest-${{ matrix.shard }}.json"
- name: Upload the shard manifest (always — the summary audits it)
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -196,7 +283,7 @@ jobs:
runs-on: ubuntu-24.04
timeout-minutes: 10
needs: [test-shard]
- if: always() && (github.event_name == 'push' || !github.event.pull_request.draft)
+ if: always() && (github.event_name != 'pull_request' || !github.event.pull_request.draft)
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml
index 108ba1016e..dabe197622 100644
--- a/.github/workflows/continuous.yml
+++ b/.github/workflows/continuous.yml
@@ -200,6 +200,16 @@ jobs:
--catalog-assets "target/${{ matrix.target }}/catalog-assets" \
--out-dir "target/${{ matrix.target }}/dist"
cat "target/${{ matrix.target }}/dist/manifest.json"
+ - name: Livecheck the assembled continuous artifact on its native runner
+ run: |
+ set -euo pipefail
+ version="$(sed -n 's/^version *= *"\([^"]*\)".*/\1/p' Cargo.toml | head -1)"
+ [[ -n "${version}" ]] || { echo "could not read the workspace version"; exit 1; }
+ python3 scripts/release/verify_release.py \
+ --dist-dir "target/${{ matrix.target }}/dist" \
+ --version "${version}" \
+ --target "${{ matrix.target }}" \
+ --sha "${GITHUB_SHA}"
- name: Upload artifacts
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
@@ -262,6 +272,16 @@ jobs:
--catalog-assets "target/${{ matrix.target }}/catalog-assets" \
--out-dir "target/${{ matrix.target }}/dist"
cat "target/${{ matrix.target }}/dist/manifest.json"
+ - name: Livecheck the assembled continuous artifact on its native runner
+ run: |
+ set -euo pipefail
+ version="$(sed -n 's/^version *= *"\([^"]*\)".*/\1/p' Cargo.toml | head -1)"
+ [[ -n "${version}" ]] || { echo "could not read the workspace version"; exit 1; }
+ python3 scripts/release/verify_release.py \
+ --dist-dir "target/${{ matrix.target }}/dist" \
+ --version "${version}" \
+ --target "${{ matrix.target }}" \
+ --sha "${GITHUB_SHA}"
- name: Upload artifacts
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b3f1aa8416..0d2b97621a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,5 +1,25 @@
# Release pipeline for Prime Agent (Rust).
#
+# THE TWO BUILD ROUTES (one promote gate):
+# - a plain vX.Y.Z tag (stable) BUILDS its own 4-target matrix: the stable
+# cadence is rare, and its cut must not depend on a continuous run the
+# staleness gate may have skipped (a superseded push builds nothing) or
+# artifact retention may have evicted;
+# - a vX.Y.Z-beta* tag (nightly) CONSUMES the continuous build's
+# artifacts: continuous already built the exact parent commit the
+# nightly tag names (nightly.yml waits out that run's green build
+# matrix before cutting the tag), so rebuilding here would pay the
+# full matrix twice per green merge for a tree that differs from its
+# parent only by the version rows. The reuse-continuous job downloads
+# that run's workflow artifacts (the download-by-run route the
+# installer's old bootstrap used) and restamps them to the beta
+# version — packaging only, no rebuild; the built bytes stay the
+# continuous build's (the restamp re-proves what it can on the runner
+# and asserts the continuous commit; the rest is continuous's own
+# green gates on that commit). Both routes upload the same
+# artifacts-<target> set into this run, so the promote gate consumes
+# one shape from either route.
+#
name: release
@@ -59,6 +79,9 @@ jobs:
build-gnu:
needs: tag-check
+ # Stable-only: a -beta* tag consumes the continuous build instead (the
+ # reuse-continuous job); exactly one of the two routes runs per tag.
+ if: ${{ !contains(github.ref_name, '-') }}
name: build-${{ matrix.target }}
runs-on: ${{ matrix.runs-on }}
# GLIBC 2.35 build baseline: artifacts built on the ubuntu-24.04 runner
@@ -197,6 +220,9 @@ jobs:
build-darwin:
needs: tag-check
+ # Stable-only: a -beta* tag consumes the continuous build instead (the
+ # reuse-continuous job); exactly one of the two routes runs per tag.
+ if: ${{ !contains(github.ref_name, '-') }}
name: build-${{ matrix.target }}
runs-on: ${{ matrix.runs-on }}
permissions:
@@ -259,8 +285,136 @@ jobs:
target/${{ matrix.target }}/dist/manifest.json
if-no-files-found: error
+ # THE BETA CHANNEL'S BUILD (the consume route): the continuous workflow
+ # already built the exact commit this beta names - nightly.yml waits out
+ # the pushed commit's continuous run (its whole build matrix green) before
+ # cutting the tag, so the tag-only commit's parent is a proven-built tip.
+ # This job resolves that parent, downloads ITS continuous run's workflow
+ # artifacts (the download-by-run route the installer's old bootstrap used),
+ # and restamps them to the beta version (scripts/release/restamp_continuous.py):
+ # the built bytes stay the continuous build's; the version-shaped facts move
+ # - the inner package.json (the runtime version manifest the shipped binary
+ # reports; the continuous stamp <version>-continuous.<sha> becomes the beta
+ # version), the archive/decoder names, and the manifest/SHA256SUMS rows (the
+ # manifest's `commit` provenance is KEPT). The result is re-uploaded as THIS
+ # run's artifacts-<target> set, so the promote gate consumes the same shape
+ # from either route. The gates the release build jobs ran on their own
+ # binaries (livecheck, GLIBC baseline) re-run here for the runner's own
+ # target; the other targets' evidence is the continuous run's own green
+ # build jobs on this exact commit (asserted via the manifest's commit row).
+ reuse-continuous:
+ needs: tag-check
+ if: ${{ contains(github.ref_name, '-') }}
+ name: consume the continuous build (no rebuild)
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ # Cross-run workflow-artifact download: the continuous run's set.
+ actions: read
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - name: Resolve the continuous run that built this beta's parent
+ id: resolve
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ set -euo pipefail
+ # THE ANNOTATED-TAG PEEL (the same ref-chain resolution the promote
+ # job's publish step uses): the tag names the tag-only commit
+ # nightly.yml created; its first parent is the continuous-built tip.
+ tag="${GITHUB_REF_NAME}"
+ ref_json="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${tag}")"
+ obj_type="$(printf '%s' "${ref_json}" | jq -r '.object.type')"
+ obj_sha="$(printf '%s' "${ref_json}" | jq -r '.object.sha')"
+ if [ "${obj_type}" = "tag" ]; then
+ tag_obj="$(gh api "repos/${GITHUB_REPOSITORY}/git/tags/${obj_sha}")"
+ obj_sha="$(printf '%s' "${tag_obj}" | jq -r '.object.sha')"
+ fi
+ parent="$(gh api "repos/${GITHUB_REPOSITORY}/git/commits/${obj_sha}" --jq '.parents[0].sha')"
+ # The run whose artifacts this beta consumes - the same query
+ # nightly.yml resolves the parent with. The cut is only created
+ # after that run concluded green with its whole build matrix,
+ # so the run must exist and be green here; anything else is a
+ # malformed tag (fail loudly, never fall back to a rebuild).
+ runs_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/continuous.yml/runs?branch=main&per_page=1&head_sha=${parent}" 2>/dev/null || echo '{"workflow_runs":[]}')"
+ read -r status conclusion run_id < <(printf '%s\n' "${runs_json}" | jq -r \
+ '(.workflow_runs[0] // {}) | [(.status // "absent"), (.conclusion // "-"), (.id // "-")] | @tsv')
+ if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then
+ echo "::error::no successful continuous run for ${parent} (${status}/${conclusion}): the beta cannot consume artifacts that do not exist - resolve the continuous run, then re-run this release"
+ exit 1
+ fi
+ echo "beta parent (the continuous-built commit): ${parent}"
+ echo "continuous run ${run_id} built ${parent}"
+ echo "parent=${parent}" >> "${GITHUB_OUTPUT}"
+ echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"
+ - name: Download the continuous build's artifacts (by run id)
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ run-id: ${{ steps.resolve.outputs.run-id }}
+ github-token: ${{ github.token }}
+ pattern: artifacts-*
+ path: incoming
+ - name: Restamp the continuous set to the beta version (no rebuild)
+ run: |
+ set -euo pipefail
+ # The promote gate consumes exactly the 4-target set; a partial
+ # download must fail here, not at the channel manifest.
+ expected="artifacts-aarch64-apple-darwin artifacts-aarch64-unknown-linux-gnu artifacts-x86_64-apple-darwin artifacts-x86_64-unknown-linux-gnu"
+ actual="$(find incoming -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort | tr '\n' ' ')"
+ if [ "${actual% }" != "${expected}" ]; then
+ echo "::error::the continuous run's artifact set is incomplete: ${actual% }"
+ exit 1
+ fi
+ python3 scripts/release/restamp_continuous.py \
+ --incoming incoming \
+ --version "${RELEASE_VERSION#v}" \
+ --expect-commit "${{ steps.resolve.outputs.parent }}" \
+ --out-dir restamped
+ - name: Upload the restamped set as this run's build artifacts
+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+ with:
+ name: artifacts-aarch64-apple-darwin
+ path: restamped/artifacts-aarch64-apple-darwin/*
+ if-no-files-found: error
+ - name: Upload the restamped set as this run's build artifacts (aarch64-unknown-linux-gnu)
+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+ with:
+ name: artifacts-aarch64-unknown-linux-gnu
+ path: restamped/artifacts-aarch64-unknown-linux-gnu/*
+ if-no-files-found: error
+ - name: Upload the restamped set as this run's build artifacts (x86_64-apple-darwin)
+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+ with:
+ name: artifacts-x86_64-apple-darwin
+ path: restamped/artifacts-x86_64-apple-darwin/*
+ if-no-files-found: error
+ - name: Upload the restamped set as this run's build artifacts (x86_64-unknown-linux-gnu)
+ uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+ with:
+ name: artifacts-x86_64-unknown-linux-gnu
+ path: restamped/artifacts-x86_64-unknown-linux-gnu/*
+ if-no-files-found: error
+
promote:
- needs: [build-gnu, build-darwin]
+ # One gate for both build routes: the stable build jobs (plain tags) or
+ # the reuse-continuous set (beta tags) upload the same artifacts-<target>
+ # set into this run. THE ROUTE GATE IS EXPLICIT: a job that needs a
+ # SKIPPED job is itself skipped unless its own `if` says otherwise (the
+ # GitHub documented default), so the default needs gate would strand
+ # promote on BOTH routes - each route deliberately skips the other two
+ # legs. The expression admits exactly the two healthy routes and nothing
+ # else: (stable) both build jobs succeeded, or (beta) the consume route
+ # succeeded; a failed leg, a cancelled run, or a failed tag-check leaves
+ # promote skipped, matching the pre-consume failure semantics. The
+ # artifact completeness assertion inside the gate is the belt behind the
+ # routing: an incomplete set fails the promotion loudly either way.
+ if: ${{ !cancelled() && (
+ (needs.build-gnu.result == 'success' && needs.build-darwin.result == 'success')
+ || needs.reuse-continuous.result == 'success') }}
+ needs: [build-gnu, build-darwin, reuse-continuous]
name: promotion gate -> release attach
runs-on: ubuntu-latest
environment: release
@@ -341,20 +495,32 @@ jobs:
import json, pathlib, tarfile
incoming = pathlib.Path("incoming")
merged, decoders, sums, version = [], [], [], None
+ commits = set()
for entry_dir in sorted(p for p in incoming.iterdir() if p.is_dir()):
manifest = json.loads((entry_dir / "manifest.json").read_text())
version = manifest["version"]
merged.extend(manifest["binaries"])
decoders.extend(manifest.get("decoders", []))
+ if manifest.get("commit"):
+ commits.add(manifest["commit"])
sums.extend(line for line in
(entry_dir / "SHA256SUMS").read_text().splitlines()
if line.strip())
+ # The consume route (beta) stamps every per-target manifest with the
+ # continuous commit it built; a set that names more than one commit
+ # is a broken restamp and must never merge into one release.
+ if len(commits) > 1:
+ raise SystemExit(f"per-target manifests name different commits: {sorted(commits)}")
merged.sort(key=lambda b: b["file"])
decoders.sort(key=lambda d: d["file"])
sums.sort(key=lambda line: line.split(None, 1)[1])
out = pathlib.Path("release-out")
out.mkdir()
result = {"version": version, "binaries": merged}
+ if commits:
+ # The build provenance rides the merged manifest (the stable
+ # rebuild route carries no commit row, as before).
+ result["commit"] = sorted(commits)[0]
if decoders:
result["decoders"] = decoders
(out / "manifest.json").write_text(
diff --git a/Makefile b/Makefile
index d8941a1458..0b87aa7276 100644
--- a/Makefile
+++ b/Makefile
@@ -165,4 +165,19 @@ catalog-assets-gates:
fold-gates:
python3 scripts/release/test_fold_changelog.py
-.PHONY: check deny windows-cross actionlint perf-wave glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates
+# The consume route's restamp battery (release.yml's beta channel): the
+# continuous artifacts restamped to the beta version keep the built bytes
+# and carry the release shape - the provenance case is first (a foreign
+# commit's artifacts are refused).
+restamp-gates:
+ python3 scripts/release/test_restamp.py
+
+# The CI shard tooling's contract battery (ci.yml's PR smoke): the stable
+# crc32 assignment under the narrowed selection, the scope-aware summary
+# audit, the selection resolver the conditional bins build reads, and the
+# fail-safe PR-files mapping the changes job feeds it.
+shard-gates:
+ python3 scripts/test_ci_test_shard.py
+ python3 scripts/test_ci_pr_crates.py
+
+.PHONY: check deny windows-cross actionlint perf-wave glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates
diff --git a/scripts/ci_pr_crates.py b/scripts/ci_pr_crates.py
new file mode 100644
index 0000000000..c96887361a
--- /dev/null
+++ b/scripts/ci_pr_crates.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Map a pull request's changed files to the crates its PR wave tests.
+
+ci.yml's `changes` job resolves the selection: the test phase narrows to the
+touched crates and their reverse dependencies (scripts/ci_test_shard.py
+--crates) on PRs; push and merge-queue waves run the full union. The
+mapping is fail-safe by construction: a selection that cannot be made is
+the FULL selection — a wrong guess can
+only cost runner minutes, never coverage.
+
+The input is the pull-requests files API (`GET /repos/{o}/{r}/pulls/{n}/files`,
+paginated by gh), one TSV row per file: `<filename>` TAB `<previous_filename>`
+(the rename sidecar; empty for ordinary rows). The API's file field is named
+`filename` (NOT `path` — the `gh pr view --json files` shape differs, and its
+100-row cap silently under-maps large PRs).
+
+The output is the job outputs (`crates` — a comma list, empty for the full
+selection — and `scope`, the human-readable decision) written to the file in
+`GITHUB_OUTPUT` (a plain report otherwise), plus a per-path report on stdout.
+
+Usage (ci.yml):
+
+ gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR}/files" \
+ --paginate --jq '.[] | [.filename, .previous_filename // ""] | @tsv' \
+ > "${RUNNER_TEMP}/pr-files.tsv"
+ python3 scripts/ci_pr_crates.py "${RUNNER_TEMP}/pr-files.tsv"
+
+Tests: python3 scripts/test_ci_pr_crates.py (make shard-gates runs it)
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+# The pulls/files API's own page ceiling: at or beyond it the list is
+# truncated and the mapping is not trustworthy, so the full selection runs.
+FILES_API_CEILING = 3000
+
+# Conservative CI impact graph: some edges cover behavior beyond a direct
+# Cargo.toml link. Over-selecting is safe; missing a dependent test is not.
+# The changes job must select without installing Rust or running Cargo.
+WORKSPACE_DEPS = {
+ "pa-types": set(),
+ "pa-telemetry": set(),
+ "pa-ai": {"pa-types"},
+ "pa-models": {"pa-ai"},
+ "pa-agent": {"pa-ai", "pa-types"},
+ "pa-core": {"pa-agent", "pa-ai", "pa-models", "pa-types", "pa-telemetry"},
+ "pa-daemon": {"pa-core"},
+ "pa-tui": {"pa-types", "pa-core"},
+ "pa-cli": {"pa-types", "pa-telemetry", "pa-ai", "pa-models",
+ "pa-agent", "pa-core", "pa-daemon", "pa-tui"},
+}
+
+
+def read_rows(path: Path) -> list[tuple[str, str]]:
+ """The files-API rows, read raw: a leading-space path (` crates/...`)
+ must stay the non-crate path it is, never be stripped into a crate
+ mapping — the fields are tab-separated TSV, @tsv-escaped by jq, so no
+ strip is needed or safe."""
+ rows: list[tuple[str, str]] = []
+ for line in path.read_text(encoding="utf-8").splitlines():
+ if not line.strip():
+ continue
+ name, _, previous = line.partition("\t")
+ rows.append((name, previous))
+ return rows
+
+
+def select_crates(rows: list[tuple[str, str]]) -> list[str] | None:
+ """The touched crates and reverse dependencies, or None for a full wave.
+
+ Only known `crates/<pkg>/**` narrows. Everything else — the workflows,
+ CI tooling, vendor/, the runtime sidecar, workspace-level Cargo files,
+ install scripts, docs — affects more than any crate subset, and a rename
+ counts both sides. An undecidable list (empty rows, the API ceiling or
+ an unknown crate) fails safe to the full selection.
+ """
+ if not rows:
+ return None
+ if len(rows) >= FILES_API_CEILING:
+ print(f"the PR has {len(rows)} changed files (the files API ceiling): "
+ "the full selection runs")
+ return None
+ crates: set[str] = set()
+ for name, previous in rows:
+ if not name:
+ # An empty filename column means the input is not the files
+ # API's shape (a field-name regression): never select from it.
+ print("an empty filename in the change list: the full selection runs")
+ return None
+ for candidate in (name, previous):
+ if not candidate:
+ continue # an ordinary (non-rename) row has no removed side
+ parts = candidate.split("/")
+ if len(parts) >= 2 and parts[0] == "crates" and parts[1]:
+ if parts[1] not in WORKSPACE_DEPS:
+ print(f"unknown crate {parts[1]}: the full selection runs")
+ return None
+ crates.add(parts[1])
+ else:
+ print(f"non-crate path {candidate}: the full selection runs")
+ return None
+ # pa-types owns the shared wire vocabulary: run every workspace test for
+ # its changes, including the independent pa-telemetry leaf crate.
+ if "pa-types" in crates:
+ return sorted(WORKSPACE_DEPS)
+ # Walk upward through the DAG until no dependent is left unselected.
+ while True:
+ dependents = {crate for crate, deps in WORKSPACE_DEPS.items()
+ if deps & crates}
+ expanded = crates | dependents
+ if expanded == crates:
+ return sorted(crates)
+ crates = expanded
+
+
+def write_outputs(crates: list[str] | None, output_path: str) -> None:
+ lines = []
+ if crates is None:
+ lines.append("crates=")
+ lines.append("scope=all")
+ else:
+ lines.append("crates=" + ",".join(crates))
+ lines.append(f"scope=crates ({len(crates)}: {', '.join(crates)})")
+ if output_path and output_path != "/dev/null":
+ with open(output_path, "a", encoding="utf-8") as handle:
+ handle.write("\n".join(lines) + "\n")
+ print("wave selection: " + lines[1].removeprefix("scope="))
+ if crates:
+ print("crates under test: " + ",".join(crates))
+
+
+def main() -> int:
+ rows = read_rows(Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/dev/stdin"))
+ crates = select_crates(rows)
+ write_outputs(crates, os.environ.get("GITHUB_OUTPUT", "/dev/null"))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/ci_test_shard.py b/scripts/ci_test_shard.py
index bac5018c0b..6193767452 100644
--- a/scripts/ci_test_shard.py
+++ b/scripts/ci_test_shard.py
@@ -33,11 +33,23 @@ recorded in the shard manifest (JSON) the summary job audits: the union of
shard manifests must equal the full enumeration, so coverage can never
silently regress when targets are added or renamed.
+The PR smoke gate narrows the selection, never the partition: `--crates
+a,b` restricts the units this wave runs to the named packages' (a PR's
+runtime smoke covers what it touched; the push wave runs the full union —
+main is the authority). The stable crc32 assignment is untouched — a unit
+still lands in exactly one shard of N — so flake attribution stays put and
+the narrow and full runs of a unit are the same leg. Every shard records the
+selection scope in its manifest, and the summary job audits the union
+against the SELECTED set: coverage can never silently regress, and a
+narrowed run can never masquerade as a full one.
+
Usage (from the repo root, e.g. in ci.yml):
python3 scripts/ci_test_shard.py --shard 2 --total 4 \
--manifest shard-manifest.json
python3 scripts/ci_test_shard.py --list # enumerate + shard assignment
+ python3 scripts/ci_test_shard.py --shard 2 --total 8 --crates pa-tui \
+ --print-selection # resolve + report the selection, run nothing
"""
from __future__ import annotations
@@ -85,6 +97,10 @@ def shard_of(unit_id: str, total: int) -> int:
return zlib.crc32(unit_id.encode("utf-8")) % total
+def _unit_package(unit_id: str) -> str:
+ return unit_id.split("#", 1)[0]
+
+
def enumerate_units() -> list[dict]:
"""The workspace's test units, exactly what `cargo test --workspace` covers.
@@ -177,15 +193,21 @@ def _failed_tests(lines: list[str]) -> list[str]:
def write_manifest(path: Path, shard: int, total: int, all_ids: list[str],
- results: list[dict]) -> None:
+ results: list[dict], crates: list[str] | None = None) -> None:
+ selection_ids = [i for i in all_ids
+ if crates is None or _unit_package(i) in set(crates)]
manifest = {
"schema": 1,
"shard": shard,
"total": total,
"all_unit_ids": all_ids,
"digest": hashlib.sha256("\n".join(all_ids).encode("utf-8")).hexdigest(),
+ "scope": scope_manifest(crates),
+ "selected_unit_ids": selection_ids,
"units": results,
- "complete": len(results) == sum(1 for i in all_ids if shard_of(i, total) == shard - 1),
+ # Complete means: every unit this wave SELECTED for this shard ran.
+ "complete": len(results) == sum(1 for i in selection_ids
+ if shard_of(i, total) == shard - 1),
}
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
@@ -193,11 +215,14 @@ def write_manifest(path: Path, shard: int, total: int, all_ids: list[str],
os.replace(tmp, path)
-def summary_md(shard: int, total: int, results: list[dict]) -> str:
+def summary_md(shard: int, total: int, results: list[dict],
+ crates: list[str] | None = None) -> str:
failed = [r for r in results if r["rc"] != 0]
+ scope = ("full selection" if crates is None
+ else f"crate selection: {', '.join(crates)}")
lines = [
f"### test shard {shard}/{total}",
- f"- {len(results) - len(failed)}/{len(results)} units green",
+ f"- {len(results) - len(failed)}/{len(results)} units green ({scope})",
]
if failed:
lines.append(f"- **{len(failed)} failed:**")
@@ -211,22 +236,81 @@ def summary_md(shard: int, total: int, results: list[dict]) -> str:
return "\n".join(lines)
+def parse_crates(raw: str | None) -> list[str] | None:
+ """The selection scope: None = the full union; a crate list narrows it."""
+ if raw is None or not raw.strip():
+ return None
+ crates = sorted({name.strip() for name in raw.split(",") if name.strip()})
+ return crates or None
+
+
+def scope_manifest(crates: list[str] | None) -> dict:
+ """The manifest's scope record (the summary audits against it)."""
+ if crates is None:
+ return {"kind": "all"}
+ return {"kind": "crates", "crates": crates}
+
+
+def selected_units(units: list[dict], crates: list[str] | None) -> list[dict]:
+ """The units this wave runs: the full union, or the named crates' subset."""
+ if crates is None:
+ return units
+ return [u for u in units if u["package"] in set(crates)]
+
+
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--shard", type=int, required=True, help="1-based shard index")
parser.add_argument("--total", type=int, required=True, help="shard count")
parser.add_argument("--manifest", type=Path, default=None,
help="write the per-shard JSON manifest here")
+ parser.add_argument("--crates", default=None,
+ help="comma-separated crate names to run (the PR smoke's "
+ "narrowed selection); omit for the full union")
+ parser.add_argument("--print-selection", action="store_true",
+ help="resolve + report this shard's selection, run nothing "
+ "(the wave's conditional bins build reads it)")
parser.add_argument("--list", action="store_true",
help="print the enumeration and shard assignment, run nothing")
args = parser.parse_args()
if not 1 <= args.shard <= args.total:
parser.error("--shard must be in 1..--total")
+ crates = parse_crates(args.crates)
+ if crates is not None and args.list:
+ parser.error("--list enumerates the full workspace; it takes no --crates")
units = enumerate_units()
+ if crates is not None:
+ # Fail closed on a mapping bug: a named package the workspace does
+ # not build would narrow the wave to a silent zero (the selection
+ # cannot name what the enumeration never saw).
+ known = {u["package"] for u in units}
+ unknown = sorted(set(crates) - known)
+ if unknown:
+ parser.error(
+ f"--crates names packages the workspace does not build: "
+ f"{', '.join(unknown)} (known: {', '.join(sorted(known))})")
all_ids = [u["id"] for u in units]
- mine = [u for u in units if shard_of(u["id"], args.total) == args.shard - 1]
- print(f"[shard {args.shard}/{args.total}] {len(mine)}/{len(units)} units")
+ selection = selected_units(units, crates)
+ mine = [u for u in selection if shard_of(u["id"], args.total) == args.shard - 1]
+ if crates is None:
+ print(f"[shard {args.shard}/{args.total}] {len(mine)}/{len(units)} units "
+ "(full selection)")
+ else:
+ print(f"[shard {args.shard}/{args.total}] {len(mine)}/{len(selection)} units "
+ f"(crate selection: {', '.join(crates)}; the full union is {len(units)})")
+
+ if args.print_selection:
+ for u in mine:
+ print(f" selected: {u['id']}")
+ # The workflow's sibling-binary build reads this (an empty selection
+ # needs no workspace bins): one machine line + the GITHUB_OUTPUT row.
+ print(f"SELECTED={len(mine)}")
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if github_output:
+ with open(github_output, "a", encoding="utf-8") as handle:
+ handle.write(f"units={len(mine)}\n")
+ return 0
if args.list:
for u in units:
@@ -243,15 +327,17 @@ def main() -> int:
f"in {result['seconds']}s")
results.append(result)
if args.manifest:
- write_manifest(args.manifest, args.shard, args.total, all_ids, results)
+ write_manifest(args.manifest, args.shard, args.total, all_ids,
+ results, crates)
if args.manifest:
- write_manifest(args.manifest, args.shard, args.total, all_ids, results)
+ write_manifest(args.manifest, args.shard, args.total, all_ids,
+ results, crates)
step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
if step_summary:
Path(step_summary).parent.mkdir(parents=True, exist_ok=True)
with open(step_summary, "a", encoding="utf-8") as f:
- f.write(summary_md(args.shard, args.total, results) + "\n")
+ f.write(summary_md(args.shard, args.total, results, crates) + "\n")
failed = [r["id"] for r in results if r["rc"] != 0]
if failed:
diff --git a/scripts/ci_test_shard_summary.py b/scripts/ci_test_shard_summary.py
index c1837d8d7a..2cb23bd274 100644
--- a/scripts/ci_test_shard_summary.py
+++ b/scripts/ci_test_shard_summary.py
@@ -1,20 +1,26 @@
#!/usr/bin/env python3
"""Audit the shard manifests and merge their failure summaries (ci.yml).
-The overall test gate is green only when EVERY shard reported and the union of
-their executed units still covers the full enumeration. This script verifies
-exactly that, so sharding can never silently drop coverage when test targets
-are added, renamed, or move packages:
+The overall test gate is green only when EVERY shard reported and the union
+of their executed units still covers the run's SELECTION — the full
+enumeration on push runs (main is the authority), or the touched crates'
+subset on the PR smoke (`--crates`, recorded by every shard as the run's
+`scope`). This script verifies exactly that, so sharding can never silently
+drop coverage when test targets are added, renamed, or move packages — and a
+narrowed run can never masquerade as a full one:
1. every shard produced a manifest;
2. all shards enumerated the same unit set (digest match);
- 3. shard assignments are disjoint and their union is the whole set;
- 4. every shard completed all of its assigned units;
- 5. every executed unit passed.
+ 3. all shards record the SAME selection scope (a mixed-scope wave is a
+ broken partition; the summary names it instead of auditing a chimera);
+ 4. shard assignments are disjoint and their union is the selected set
+ (nothing selected is skipped, nothing outside the selection ran);
+ 5. every shard completed all of its selected units;
+ 6. every executed unit passed.
-It then prints ONE merged report: which binaries failed, in which shard, with
-their failing test names — the single place a lane looks when a PR run goes
-red, instead of crawling four job logs.
+It then prints ONE merged report: the scope, which binaries failed, in which
+shard, with their failing test names — the single place a lane looks when a
+PR run goes red, instead of crawling the job logs.
Usage (from the repo root, in ci.yml's test summary job):
@@ -44,6 +50,32 @@ def load_manifests(manifest_dir: Path, total: int):
return manifests
+def run_scope(manifests: dict) -> tuple[str, list[str] | None]:
+ """The wave's selection scope, from the manifests (one source of truth)."""
+ scopes = {json.dumps(m.get("scope", {"kind": "all"}), sort_keys=True)
+ for m in manifests.values()}
+ if len(scopes) != 1:
+ return "MIXED", None
+ scope = json.loads(next(iter(scopes)))
+ return scope.get("kind", "all"), scope.get("crates")
+
+
+def selected_ids(manifest: dict, crates: list[str] | None) -> list[str]:
+ """The selected universe: the manifest's recorded selection, or (older
+ manifests without one) the full enumeration."""
+ if "selected_unit_ids" in manifest:
+ return manifest["selected_unit_ids"]
+ all_ids = manifest["all_unit_ids"]
+ if crates is None:
+ return all_ids
+ packages = set(crates)
+ return [i for i in all_ids if _unit_package(i) in packages]
+
+
+def _unit_package(unit_id: str) -> str:
+ return unit_id.split("#", 1)[0]
+
+
def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:
"""Structural audit failures and the set of failing unit ids."""
problems = []
@@ -63,6 +95,17 @@ def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:
return problems, failed_units
all_ids = manifests[1]["all_unit_ids"]
+ kind, crates = run_scope(manifests)
+ if kind == "MIXED":
+ described = sorted(json.dumps(m.get("scope"), sort_keys=True)
+ for m in manifests.values())
+ problems.append("shards recorded different selection scopes — a "
+ "mixed-scope wave is a broken partition: " +
+ ", ".join(described))
+ return problems, failed_units
+ selection = selected_ids(manifests[1], crates)
+ selection_set = set(selection)
+
executed: dict[str, str] = {} # unit id -> shard that ran it
for shard, manifest in sorted(manifests.items()):
for unit in manifest["units"]:
@@ -78,16 +121,19 @@ def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:
if unit["rc"] != 0:
failed_units.add(uid)
- missing = sorted(set(all_ids) - set(executed))
+ missing = sorted(selection_set - set(executed))
if missing:
- problems.append(f"unassigned units (no shard ran them): {missing}")
- extra = sorted(set(executed) - set(all_ids))
+ problems.append(f"selected units no shard ran: {missing}")
+ extra = sorted(set(executed) - selection_set)
if extra:
- problems.append(f"assigned outside the enumeration: {extra}")
+ problems.append(f"units outside the selection ran: {extra}")
+ outside = sorted(set(executed) - set(all_ids))
+ if outside:
+ problems.append(f"assigned outside the enumeration: {outside}")
for shard, manifest in sorted(manifests.items()):
if not manifest.get("complete", False):
- assigned = [i for i in all_ids if shard_of(i, total) == shard - 1]
+ assigned = [i for i in selection if shard_of(i, total) == shard - 1]
unfinished = sorted(set(assigned) - {u["id"] for u in manifest["units"]})
problems.append(f"shard {shard} is incomplete; units it never "
f"reported: {unfinished}")
@@ -96,7 +142,13 @@ def audit(manifests: dict, total: int) -> tuple[list[str], set[str]]:
def merged_report(manifests: dict, total: int, problems: list[str],
failed_units: set[str]) -> str:
- lines = [f"### test summary ({total} shards)"]
+ kind, crates = run_scope(manifests)
+ scope = ("full selection" if kind == "all"
+ else f"crate selection: {', '.join(crates or [])}")
+ selected = selected_ids(manifests.get(1, {"all_unit_ids": []}), crates)
+ union_size = len(manifests.get(1, {}).get("all_unit_ids", []))
+ lines = [f"### test summary ({total} shards)",
+ f"- scope: {scope} — {len(selected)} of {union_size} units selected"]
for shard in sorted(manifests):
manifest = manifests[shard]
units = manifest["units"]
diff --git a/scripts/release/restamp_continuous.py b/scripts/release/restamp_continuous.py
new file mode 100644
index 0000000000..c43f8c5c64
--- /dev/null
+++ b/scripts/release/restamp_continuous.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""Restamp the continuous build's artifacts into a release-channel set.
+
+The beta (nightly) release consumes the `continuous` workflow's artifacts
+instead of rebuilding the 4-target matrix: `continuous` already built the
+exact parent commit the nightly tag names (its own gates — GLIBC baseline,
+split-debug, payload guards — ran green there, and nightly.yml waits out
+that run before cutting the tag). The only differences between a continuous
+artifact set and a release one are version-shaped, so this script restamps
+them without touching the built bytes:
+
+ - the tarball's inner `package.json` (the runtime version manifest the
+ shipped binary reports via `--version`; see pa-cli config::version)
+ moves from `<version>-continuous.<sha>` to the release version, keeping
+ the `commit` provenance;
+ - the archive, its SHA256SUMS line, and the (Linux) split-debug decoder
+ rename to the release version;
+ - the per-target manifest.json moves to the release version (per-binary
+ and decoder rows kept in sync; `executableSha256`/`buildId` unchanged —
+ the binaries are the continuous build's).
+
+Everything else is verified, never rebuilt: the incoming archive must match
+its own SHA256SUMS + manifest (an intact download), carry exactly the
+continuous payload (assemble's designed entries plus `package.json`), be
+stamped with the expected commit, and keep its decoder matching the
+shipped ELF. Deterministic repacking reuses assemble_artifacts's own packer,
+so the restamped archive is byte-equivalent to one assembled at the release
+version from the same tree.
+
+Usage (release.yml's consume job):
+
+ python3 scripts/release/restamp_continuous.py \
+ --incoming <dir with artifacts-<target>/ subdirs> \
+ --version <x.y.z[-beta.N]> \
+ --expect-commit <40-hex sha> \
+ --out-dir <dir>
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+
+# Same-directory release scripts (the packer + gates are reused, never re-implemented).
+from assemble_artifacts import (
+ TARGET_ALIASES,
+ fail_if_decoder_in_archive,
+ pack_tarball,
+ sha256_file,
+)
+from bundle_catalog import validate_bundled_catalog_dir
+from verify_release import CONTINUOUS_EXTRA_TOP_LEVEL, CONTINUOUS_SUFFIX, EXPECTED_TOP_LEVEL
+
+# The runner-host target triple -> the binary this process can execute
+# (installer.rs `target_for` parity: the same four continuous targets).
+HOST_TARGETS = {
+ ("linux", "x86_64"): "x86_64-unknown-linux-gnu",
+ ("linux", "aarch64"): "aarch64-unknown-linux-gnu",
+ ("macos", "aarch64"): "aarch64-apple-darwin",
+ ("macos", "x86_64"): "x86_64-apple-darwin",
+}
+
+
+def fail(message: str) -> None:
+ print(f"error: {message}", file=sys.stderr)
+ sys.exit(1)
+
+
+def host_target() -> str | None:
+ return HOST_TARGETS.get((platform.system().lower(), platform.machine()))
+
+
+def read_sums(path: Path) -> dict[str, str]:
+ """SHA256SUMS rows as {basename: digest}."""
+ sums: dict[str, str] = {}
+ for line in path.read_text(encoding="utf-8").splitlines():
+ if not line.strip():
+ continue
+ digest, name = line.split(None, 1)
+ sums[Path(name.strip()).name] = digest.strip()
+ return sums
+
+
+def _member_target(archive: Path, staging: Path, member: tarfile.TarInfo) -> Path:
+ """The extraction path for one member, proven inside `staging` first.
+
+ A member name with an absolute path or a `..` component would write
+ OUTSIDE the staging tree (the tarfile tar-slip class); the payload gate
+ below runs only after the whole archive is read, so the traversal check
+ belongs here, before any byte is written.
+ """
+ candidate = staging / member.name
+ if member.name.startswith("/") or ".." in Path(member.name).parts:
+ fail(f"archive {archive.name} carries an escaping member {member.name!r}; "
+ "refusing to extract outside the staging tree")
+ resolved = candidate.resolve()
+ if staging.resolve() not in resolved.parents and resolved != staging.resolve():
+ fail(f"archive {archive.name} carries an escaping member {member.name!r}; "
+ "refusing to extract outside the staging tree")
+ return candidate
+
+
+def unpack_archive(archive: Path, staging: Path) -> list[str]:
+ """Extract a plain-file archive to `staging`; return its top-level entries."""
+ with tarfile.open(archive, "r:gz") as tar:
+ members = tar.getmembers()
+ for member in members:
+ if member.issym() or member.islnk():
+ fail(f"archive {archive.name} contains a link entry {member.name!r}; "
+ "refusing to restamp a payload the packer never produces")
+ target = _member_target(archive, staging, member)
+ if member.isdir():
+ target.mkdir(parents=True, exist_ok=True)
+ else:
+ target.parent.mkdir(parents=True, exist_ok=True)
+ source = tar.extractfile(member)
+ assert source is not None
+ with source, open(target, "wb") as sink:
+ shutil.copyfileobj(source, sink)
+ target.chmod(member.mode)
+ return sorted({member.name.split("/", maxsplit=1)[0] for member in members})
+
+
+def glibc_baseline_ok(binary: Path) -> bool:
+ """No symbol above GLIBC_2.35 (the Ubuntu 22.04 release baseline).
+
+ The continuous build gated this inside its build container; the restamp
+ re-proves it on the host it can execute.
+ """
+ result = subprocess.run(["objdump", "-T", str(binary)],
+ capture_output=True, text=True, check=False)
+ if result.returncode != 0:
+ return False
+ versions = [tuple(int(part) for part in symbol.split("_", 1)[1].split("."))
+ for symbol in re.findall(r"GLIBC_[0-9.]+", result.stdout)]
+ return bool(versions) and max(versions) <= (2, 35)
+
+
+def restamp_target(incoming: Path, out_dir: Path, version: str,
+ expect_commit: str) -> dict:
+ manifest = json.loads((incoming / "manifest.json").read_text(encoding="utf-8"))
+ binaries = manifest["binaries"]
+ if len(binaries) != 1:
+ fail(f"{incoming.name}: expected exactly one binary row, found {len(binaries)}")
+ binary = binaries[0]
+ target = binary["target"]
+ alias = TARGET_ALIASES[target]
+ staged_commit = str(manifest.get("commit", "")).lower()
+ if staged_commit != expect_commit:
+ fail(f"{incoming.name}: artifact commit {staged_commit!r} != the tag's "
+ f"parent {expect_commit!r}: these are not the continuous builds "
+ "of the commit this release names")
+ old_version = manifest["version"].removeprefix("v")
+ old_archive_name = binary["file"]
+
+ # 1. The download is intact: its own SHA256SUMS + manifest must agree.
+ archive = incoming / old_archive_name
+ if not archive.is_file():
+ fail(f"{incoming.name}: archive {old_archive_name} missing")
+ archive_sha = sha256_file(archive)
+ sums = read_sums(incoming / "SHA256SUMS")
+ if sums.get(old_archive_name) != archive_sha or binary["sha256"] != archive_sha:
+ fail(f"{incoming.name}: {old_archive_name} does not match its "
+ "SHA256SUMS/manifest digest — the download is not intact")
+
+ # 2. The decoder rows (Linux) must be present and match their files.
+ decoders = manifest.get("decoders", [])
+ decoder_files = sorted(incoming.glob("*.debug.gz"))
+ if target.endswith("-unknown-linux-gnu"):
+ if len(decoders) != 1 or len(decoder_files) != 1:
+ fail(f"{incoming.name}: a Linux target requires exactly one decoder "
+ f"(manifest rows {len(decoders)}, files {len(decoder_files)})")
+ if sha256_file(decoder_files[0]) != decoders[0]["sha256"]:
+ fail(f"{incoming.name}: the decoder does not match its manifest sha256")
+ if sums.get(decoder_files[0].name) != decoders[0]["sha256"]:
+ fail(f"{incoming.name}: the decoder's SHA256SUMS line does not match")
+ elif decoders or decoder_files:
+ fail(f"{incoming.name}: unexpected decoder artifacts on {target}")
+
+ # 3. Unpack; assert the continuous payload shape.
+ staging = Path(tempfile.mkdtemp(prefix="prime-agent-restamp-"))
+ try:
+ entries = unpack_archive(archive, staging)
+ expected = sorted(EXPECTED_TOP_LEVEL | CONTINUOUS_EXTRA_TOP_LEVEL)
+ if entries != expected:
+ fail(f"{incoming.name}: payload {entries} != the continuous "
+ f"payload {expected}; refusing to restamp a foreign archive")
+ package = json.loads((staging / "package.json").read_text(encoding="utf-8"))
+ stamped = f"{old_version}-{CONTINUOUS_SUFFIX}.{expect_commit}"
+ if package.get("version") != stamped:
+ fail(f"{incoming.name}: package.json version "
+ f"{package.get('version')!r} != the continuous stamp "
+ f"{stamped!r}: not a continuous build's artifact")
+ if sha256_file(staging / "prime-agent") != binary["executableSha256"]:
+ fail(f"{incoming.name}: the staged binary does not match the "
+ "manifest's executableSha256")
+ validate_bundled_catalog_dir(staging)
+
+ # 4. The runtime version manifest moves to the release version; the
+ # built bytes stay the continuous build's.
+ package["version"] = version
+ (staging / "package.json").write_text(json.dumps(package, indent=2) + "\n")
+
+ # 5. Repack deterministically (the same packer that built the archive).
+ new_archive_name = f"prime-agent-{version}-{alias}.tar.gz"
+ target_dir = out_dir / incoming.name
+ target_dir.mkdir(parents=True, exist_ok=True)
+ new_archive = target_dir / new_archive_name
+ pack_tarball(staging, new_archive, entries)
+ fail_if_decoder_in_archive(new_archive)
+ new_sha = sha256_file(new_archive)
+
+ # 6. The decoder rides along unchanged (bytes), renamed to the release version.
+ new_lines = [f"{new_sha} {new_archive_name}"]
+ if target.endswith("-unknown-linux-gnu"):
+ new_decoder_name = f"prime-agent-{version}-{alias}.debug.gz"
+ shutil.copyfile(decoder_files[0], target_dir / new_decoder_name)
+ new_lines.append(f"{decoders[0]['sha256']} {new_decoder_name}")
+ decoders[0]["file"] = new_decoder_name
+ (target_dir / "SHA256SUMS").write_text("\n".join(new_lines) + "\n")
+
+ # 7. The per-target manifest moves to the release version, keeping
+ # the commit provenance of the build it consumed.
+ binary["version"] = f"v{version}"
+ binary["file"] = new_archive_name
+ binary["sha256"] = new_sha
+ manifest["version"] = f"v{version}"
+ (target_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+
+ # 8. The live gates, on the target this host can execute: the shipped
+ # binary's `--version` must report the release version (the runtime
+ # version manifest is what the update flow reads), and a GNU target
+ # re-proves the GLIBC baseline. The other targets' binaries cannot
+ # execute here; verify_release.py livechecked each on its native
+ # continuous runner (the manifest's `commit` row names that build).
+ livecheck = None
+ if target == host_target():
+ env = {k: v for k, v in os.environ.items() if k != "PI_PACKAGE_DIR"}
+ run = subprocess.run([str(staging / "prime-agent"), "--version"],
+ capture_output=True, text=True, env=env,
+ cwd=staging, check=False)
+ detail = (run.stdout or run.stderr).strip() or f"rc={run.returncode}"
+ if run.returncode != 0 or run.stdout.strip() != version:
+ fail(f"{incoming.name}: the restamped binary reports {detail!r}, "
+ f"expected {version!r} — the runtime version manifest did not take")
+ if target.endswith("-unknown-linux-gnu") \
+ and not glibc_baseline_ok(staging / "prime-agent"):
+ fail(f"{incoming.name}: the binary requires symbols above "
+ "GLIBC_2.35 — above the Ubuntu 22.04 release baseline")
+ livecheck = version
+ return {
+ "target": target,
+ "platform": alias,
+ "file": new_archive_name,
+ "sha256": new_sha,
+ "commit": expect_commit,
+ "executableSha256": binary["executableSha256"],
+ "livecheck": livecheck,
+ }
+ finally:
+ shutil.rmtree(staging, ignore_errors=True)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--incoming", required=True, type=Path,
+ help="directory holding the downloaded artifacts-<target> sets")
+ parser.add_argument("--version", required=True,
+ help="the release version to stamp (e.g. 0.9.8-beta.1)")
+ parser.add_argument("--expect-commit", required=True,
+ help="the 40-hex commit the artifacts must be stamped with")
+ parser.add_argument("--out-dir", required=True, type=Path,
+ help="directory to write the restamped artifacts-<target> sets to")
+ args = parser.parse_args()
+ if not re.fullmatch(r"[0-9a-f]{40}", args.expect_commit):
+ fail(f"invalid commit {args.expect_commit!r} (expected 40 hex chars)")
+ if not re.fullmatch(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?", args.version):
+ fail(f"invalid version {args.version!r}")
+
+ incoming = [p for p in sorted(args.incoming.iterdir()) if p.is_dir()]
+ if not incoming:
+ fail(f"no artifacts-* directories under {args.incoming}")
+ results = [restamp_target(directory, args.out_dir, args.version,
+ args.expect_commit.lower())
+ for directory in incoming]
+ for result in results:
+ print(json.dumps(result, indent=2))
+ livechecked = [r["target"] for r in results if r["livecheck"]]
+ print(f"restamped {len(results)} target sets to {args.version} "
+ f"(continuous commit {args.expect_commit}; "
+ f"livechecked: {', '.join(livechecked) or 'none on this host'}); "
+ "the built bytes are unchanged")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/release/test_restamp.py b/scripts/release/test_restamp.py
new file mode 100644
index 0000000000..d244760390
--- /dev/null
+++ b/scripts/release/test_restamp.py
@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+"""Test battery for the continuous-artifact restamp (the consume route's
+packaging half, release.yml's beta channel).
+
+The beta release consumes the continuous build's artifacts and restamps
+them to the beta version (scripts/release/restamp_continuous.py): no
+rebuild, the built bytes stay the continuous build's. This battery pins the
+restamp's contracts on a SYNTHETIC continuous artifact set built by the
+real assembler (`assemble_artifacts.py --sha`, the exact shape continuous.yml
+uploads), with the security case first: the restamp must refuse anything
+that is not the named commit's continuous build.
+
+The fixture binary mirrors the shipped runtime contract the restamp
+relies on (pa-cli config::version): it prints the version from the
+exe-adjacent `package.json` at runtime, falling back to a compiled-in
+version when the manifest is unreadable — so the livecheck proves the
+version restamp through the same mechanism the update flow reads.
+
+Run: python3 scripts/release/test_restamp.py (or: make restamp-gates)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parent
+REPO = SCRIPTS_DIR.parent.parent
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from assemble_artifacts import TARGET_ALIASES # noqa: E402 (same release-scripts directory)
+
+ASSEMBLER = SCRIPTS_DIR / "assemble_artifacts.py"
+SPLITTER = SCRIPTS_DIR / "split_debug.py"
+BUNDLER = SCRIPTS_DIR / "bundle_catalog.py"
+RESTAMPER = SCRIPTS_DIR / "restamp_continuous.py"
+DECODER_VERIFIER = SCRIPTS_DIR / "verify_decoders.py"
+VERIFIER = SCRIPTS_DIR / "verify_release.py"
+
+HOST_TARGET = "x86_64-unknown-linux-gnu"
+HOST_ALIAS = TARGET_ALIASES[HOST_TARGET]
+DARWIN_TARGET = "x86_64-apple-darwin"
+DARWIN_ALIAS = TARGET_ALIASES[DARWIN_TARGET]
+
+# The runtime version resolver the shipped binary implements
+# (pa-cli config::version parity: the exe-adjacent package.json wins at
+# runtime; the compiled-in version is the fallback when the manifest is
+# missing or malformed).
+VERSION_RESOLVER_C = r'''#include <stdio.h>
+#include <string.h>
+
+int main(int argc, char **argv) {
+ if (argc > 1 && strcmp(argv[1], "--version") == 0) {
+ FILE *manifest = fopen("package.json", "r");
+ if (manifest) {
+ char line[512];
+ while (fgets(line, sizeof(line), manifest)) {
+ char *key = strstr(line, "\"version\"");
+ if (!key) continue;
+ char *start = strchr(key + 10, '"');
+ if (!start) break;
+ char *end = strchr(start + 1, '"');
+ if (!end) break;
+ fwrite(start + 1, 1, end - start - 1, stdout);
+ putchar('\n');
+ fclose(manifest);
+ return 0;
+ }
+ fclose(manifest);
+ }
+ puts("%s");
+ return 0;
+ }
+ puts("prime-agent (test fixture)");
+ return 0;
+}
+'''
+
+
+def run_cli(script, args, cwd=None):
+ return subprocess.run([sys.executable, str(script), *args],
+ capture_output=True, text=True, cwd=cwd or REPO,
+ check=False)
+
+
+def digest(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+class ContinuousFixture:
+ """A synthetic continuous artifact set, the exact shape continuous.yml
+ uploads: a real (tiny) ELF with a split-debug decoder, assembled by the
+ real assembler with the commit stamp."""
+
+ def __init__(self, root: Path, version: str, sha: str):
+ self.root = root
+ self.version = version
+ self.sha = sha
+ (root / "skills").mkdir(parents=True)
+ (root / "skills" / "skill.md").write_text("# skill\n")
+ (root / "LICENSE").write_text("license\n")
+ (root / "README.md").write_text("readme\n")
+ runtime = root / "runtime"
+ runtime.mkdir()
+ (runtime / "pyproject.toml").write_text("[project]\nname='rlm'\n")
+ (runtime / "src").mkdir()
+ (root / "prime-agent-runtime").mkdir()
+ (root / "prime-agent-runtime" / "pyproject.toml").write_text("[project]\nname='rlm'\n")
+
+ source = root / "resolver.c"
+ source.write_text(VERSION_RESOLVER_C % version)
+ self.raw = root / "bin" / "cargo-prime-agent"
+ self.raw.parent.mkdir()
+ subprocess.run(["gcc", "-g", "-Wl,--build-id", "-o", str(self.raw),
+ str(source)], check=True)
+ self.binary = root / "bin" / "prime-agent"
+ self.decoder = root / "bin" / f"prime-agent-{version}-{HOST_ALIAS}.debug.gz"
+ split = run_cli(SPLITTER, ["--binary", str(self.raw), "--shipped",
+ str(self.binary), "--out", str(self.binary.parent),
+ "--version", version, "--target", HOST_TARGET])
+ if split.returncode:
+ raise AssertionError(split.stderr)
+ self.catalog = root / "catalog-assets"
+ bundled = run_cli(BUNDLER, ["generate", "--fixture", "--out", str(self.catalog)])
+ if bundled.returncode:
+ raise AssertionError(bundled.stderr)
+ self.dist = root / "dist"
+ self.dist.mkdir()
+ self.assemble_into(self.dist, sha=sha)
+
+ def assemble_into(self, out_dir: Path, sha: str | None, target: str = HOST_TARGET):
+ args = ["--repo-root", str(self.root), "--version", self.version,
+ "--target", target, "--binary", str(self.binary),
+ "--runtime-dir", str(self.root / "runtime"),
+ "--out-dir", str(out_dir)]
+ if target.endswith("-unknown-linux-gnu"):
+ args += ["--decoder", str(self.decoder)]
+ if sha:
+ args += ["--sha", sha]
+ args += ["--catalog-assets", str(self.catalog)]
+ result = run_cli(ASSEMBLER, args)
+ if result.returncode:
+ raise AssertionError(result.stderr)
+ # The CI build jobs point --decoder at the dist tree (split_debug
+ # writes it there), so the decoder rides the assembled set; the
+ # fixture mirrors that layout.
+ if target.endswith("-unknown-linux-gnu"):
+ shutil.copyfile(self.decoder,
+ out_dir / f"prime-agent-{self.version}-{HOST_ALIAS}.debug.gz")
+ return out_dir
+
+ def artifact_dir(self, out_dir: Path, target: str = HOST_TARGET) -> Path:
+ """The per-target layout continuous.yml uploads (download-artifact
+ places one directory per artifact when a run has more than one)."""
+ dist = out_dir / f"artifacts-{target}"
+ dist.mkdir()
+ archive = out_dir / f"prime-agent-{self.version}-{TARGET_ALIASES[target]}.tar.gz"
+ shutil.move(str(archive), str(dist / archive.name))
+ for name in ("SHA256SUMS", "manifest.json"):
+ shutil.move(str(out_dir / name), str(dist / name))
+ if target.endswith("-unknown-linux-gnu"):
+ decoder = out_dir / f"prime-agent-{self.version}-{HOST_ALIAS}.debug.gz"
+ shutil.move(str(decoder), str(dist / decoder.name))
+ return dist
+
+
+class RestampTestCase(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+ self.commit = "a" * 40
+ self.fixture = ContinuousFixture(self.root / "repo", "9.9.9", self.commit)
+ self.incoming = self.root / "incoming"
+ shutil.copytree(self.fixture.artifact_dir(self.fixture.dist),
+ self.incoming / f"artifacts-{HOST_TARGET}")
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def restamp(self, version="9.9.9-beta.2", commit=None, expect=0):
+ out = self.root / "restamped"
+ result = run_cli(RESTAMPER, [
+ "--incoming", str(self.incoming), "--version", version,
+ "--expect-commit", commit or self.commit, "--out-dir", str(out)])
+ self.assertEqual(result.returncode, expect,
+ f"restamp rc={result.returncode}: {result.stderr}")
+ return result, out
+
+
+class TheProvenanceCase(RestampTestCase):
+ """The restamp consumes ONLY the named commit's continuous build."""
+
+ def test_a_foreign_commit_is_refused(self):
+ result, _ = self.restamp(commit="b" * 40, expect=1)
+ self.assertIn("not the continuous builds of the commit", result.stderr)
+
+ def test_a_release_build_without_the_continuous_stamp_is_refused(self):
+ plain = self.root / "plain-dist"
+ plain.mkdir()
+ self.fixture.assemble_into(plain, sha=None) # a release-shaped set, no package.json
+ shutil.rmtree(self.incoming / f"artifacts-{HOST_TARGET}")
+ shutil.copytree(self.fixture.artifact_dir(plain), self.incoming / f"artifacts-{HOST_TARGET}")
+ result, _ = self.restamp(expect=1)
+ # The provenance gate fires first (a release-shaped set carries no
+ # commit row); either refusal names the missing continuous stamp.
+ self.assertTrue(
+ "not the continuous builds of the commit" in result.stderr
+ or "not a continuous build's artifact" in result.stderr,
+ result.stderr)
+
+ def test_a_tampered_archive_is_refused(self):
+ archive = self.incoming / f"artifacts-{HOST_TARGET}" / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"
+ data = bytearray(archive.read_bytes())
+ data[-1] ^= 0xFF
+ archive.write_bytes(bytes(data))
+ result, _ = self.restamp(expect=1)
+ self.assertIn("does not match its SHA256SUMS", result.stderr)
+
+
+class TheRestampCase(RestampTestCase):
+ """The restamped set is the release shape at the beta version, built
+ bytes unchanged."""
+
+ def test_the_continuous_artifact_passes_its_native_livecheck(self):
+ # The continuous matrix now runs this verifier on every native runner
+ # before uploading: exercise its --sha path on the assembled set.
+ verified = run_cli(VERIFIER, [
+ "--dist-dir", str(self.incoming / f"artifacts-{HOST_TARGET}"),
+ "--version", self.fixture.version, "--target", HOST_TARGET,
+ "--sha", self.commit])
+ self.assertEqual(verified.returncode, 0, verified.stderr)
+ self.assertIn("livecheck all OK", verified.stdout)
+
+ def restamped_assertions(self, out: Path, version: str):
+ target_dir = out / f"artifacts-{HOST_TARGET}"
+ archive = target_dir / f"prime-agent-{version}-{HOST_ALIAS}.tar.gz"
+ self.assertTrue(archive.is_file(), "the restamped archive is renamed to the version")
+ manifest = json.loads((target_dir / "manifest.json").read_text())
+ self.assertEqual(manifest["version"], f"v{version}")
+ self.assertEqual(manifest["commit"], self.commit)
+ binary = manifest["binaries"][0]
+ self.assertEqual(binary["file"], archive.name)
+ self.assertEqual(binary["sha256"], digest(archive))
+ self.assertEqual(binary["executableSha256"],
+ json.loads((self.incoming / f"artifacts-{HOST_TARGET}" / "manifest.json")
+ .read_text())["binaries"][0]["executableSha256"])
+ sums = {line.split()[1]: line.split()[0]
+ for line in (target_dir / "SHA256SUMS").read_text().splitlines() if line.strip()}
+ decoder_name = f"prime-agent-{version}-{HOST_ALIAS}.debug.gz"
+ self.assertEqual(set(sums), {archive.name, decoder_name})
+ # The decoder's bytes are the continuous build's, only renamed.
+ old_decoder = self.incoming / f"artifacts-{HOST_TARGET}" / f"prime-agent-9.9.9-{HOST_ALIAS}.debug.gz"
+ self.assertEqual(digest(target_dir / decoder_name), digest(old_decoder))
+ self.assertEqual(manifest["decoders"][0]["file"], decoder_name)
+ # The inner runtime version manifest is what the update flow reads.
+ with tarfile.open(archive) as tar:
+ package = json.load(tar.extractfile("package.json"))
+ self.assertEqual(package["version"], version)
+ self.assertEqual(package["commit"], self.commit)
+ return target_dir
+
+ def test_the_beta_version_lands_end_to_end(self):
+ version = "9.9.9-beta.2"
+ result, out = self.restamp(version=version)
+ self.restamped_assertions(out, version)
+ # The shipped binary reports the beta version through the package.json
+ # the restamp rewrote (config::version parity: the exe-adjacent
+ # manifest wins at runtime), and the promote-side decoder gate passes
+ # on the restamped set — the two integration seams of the consume route.
+ self.assertIn(f'"livecheck": "{version}"', result.stdout)
+ verify = run_cli(DECODER_VERIFIER, [str(out)])
+ self.assertEqual(verify.returncode, 0, verify.stderr)
+ self.assertIn("OK", verify.stdout)
+ # The release verifier accepts the restamped shape with its exact
+ # gate set: payload, checksums, manifest, catalog assets, livecheck.
+ target_dir = out / f"artifacts-{HOST_TARGET}"
+ verified = run_cli(VERIFIER, [
+ "--dist-dir", str(target_dir), "--version", version,
+ "--target", HOST_TARGET, "--expect-package-json", version])
+ self.assertEqual(verified.returncode, 0, verified.stderr)
+
+ def test_a_non_gnu_target_restamps_without_a_decoder(self):
+ darwin = self.fixture.assemble_into(self.root / "darwin-dist", sha=self.commit,
+ target=DARWIN_TARGET)
+ shutil.copytree(self.fixture.artifact_dir(darwin, target=DARWIN_TARGET),
+ self.incoming / f"artifacts-{DARWIN_TARGET}")
+ result, out = self.restamp(version="9.9.9-beta.7")
+ target_dir = out / f"artifacts-{DARWIN_TARGET}"
+ archive = target_dir / f"prime-agent-9.9.9-beta.7-{DARWIN_ALIAS}.tar.gz"
+ self.assertTrue(archive.is_file())
+ manifest = json.loads((target_dir / "manifest.json").read_text())
+ self.assertEqual(manifest["version"], "v9.9.9-beta.7")
+ self.assertNotIn("decoders", manifest)
+ sums = (target_dir / "SHA256SUMS").read_text().splitlines()
+ self.assertEqual(len(sums), 1)
+ # The foreign-arch set cannot execute on this host: restamp skips its
+ # livecheck; continuous verified it earlier on its native runner.
+ self.assertIn('"livecheck": null', result.stdout)
+
+ def test_an_escaping_tar_member_is_refused_before_any_write(self):
+ """The tar-slip class: a member named `..`/absolute must never be
+ extracted outside the staging tree (the payload gate runs only
+ after the unpack, so the traversal guard is the unpack's own)."""
+ import tarfile
+ staging = self.root / "sneak"
+ staging.mkdir()
+ target = self.incoming / f"artifacts-{HOST_TARGET}"
+ archive = target / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"
+ with tarfile.open(archive) as tar:
+ tar.extractall(staging)
+ malicious = staging / "prime-agent-9.9.9-evil.tar.gz"
+ with tarfile.open(malicious, "w:gz") as tar:
+ info = tarfile.TarInfo("../escaped-payload")
+ payload = b"outside the staging tree"
+ info.size = len(payload)
+ tar.addfile(info, __import__("io").BytesIO(payload))
+ import hashlib
+ new_sha = hashlib.sha256(malicious.read_bytes()).hexdigest()
+ shutil.move(str(malicious), str(archive))
+ sums = (target / "SHA256SUMS").read_text().splitlines()
+ (target / "SHA256SUMS").write_text("\n".join(
+ new_sha + line[64:] if line.split()[1] == archive.name else line
+ for line in sums) + "\n")
+ manifest = json.loads((target / "manifest.json").read_text())
+ manifest["binaries"][0]["sha256"] = new_sha
+ (target / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+ result, _ = self.restamp(expect=1)
+ self.assertIn("escaping member", result.stderr)
+ self.assertFalse((self.root / "escaped-payload").exists(),
+ "no byte may land outside the staging tree")
+
+ def test_repacking_is_deterministic(self):
+ _, first = self.restamp(version="9.9.9-beta.3")
+ first_archive = digest(first / f"artifacts-{HOST_TARGET}" /
+ f"prime-agent-9.9.9-beta.3-{HOST_ALIAS}.tar.gz")
+ _, second = self.restamp(version="9.9.9-beta.3")
+ second_archive = digest(second / f"artifacts-{HOST_TARGET}" /
+ f"prime-agent-9.9.9-beta.3-{HOST_ALIAS}.tar.gz")
+ self.assertEqual(first_archive, second_archive)
+
+ def test_a_foreign_payload_is_refused(self):
+ # Sneak an unexpected top-level file into the set and rebuild the
+ # archive deterministically; the sums stay stale, so this also proves
+ # the integrity gate fires before the payload gate ever matters.
+ staging = self.root / "sneak"
+ staging.mkdir()
+ archive = self.incoming / f"artifacts-{HOST_TARGET}" / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"
+ with tarfile.open(archive) as tar:
+ tar.extractall(staging)
+ (staging / "EXTRA").write_text("unexpected\n")
+ entries = sorted(p.name for p in staging.iterdir())
+ from assemble_artifacts import pack_tarball
+ rebuilt = staging / f"prime-agent-9.9.9-{HOST_ALIAS}.tar.gz"
+ pack_tarball(staging, rebuilt, entries)
+ shutil.move(str(rebuilt), str(archive))
+ result, _ = self.restamp(expect=1)
+ self.assertTrue("does not match its SHA256SUMS" in result.stderr
+ or "!= the continuous payload" in result.stderr,
+ result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/scripts/release/verify_release.py b/scripts/release/verify_release.py
index 8a1c8c0664..84f2cb20da 100644
--- a/scripts/release/verify_release.py
+++ b/scripts/release/verify_release.py
@@ -82,12 +82,19 @@ def main() -> int:
parser.add_argument("--target", required=True)
parser.add_argument("--sha", default=None,
help="commit SHA the tarball must be stamped with")
+ parser.add_argument("--expect-package-json", default=None, metavar="VERSION",
+ help="the consume-route restamp shape: a package.json "
+ "carrying exactly this release version")
args = parser.parse_args()
if args.sha is not None:
args.sha = args.sha.lower()
+ if args.expect_package_json and args.sha:
+ fail("--sha (the continuous stamp) and --expect-package-json "
+ "(the restamped release shape) are mutually exclusive")
expected_top_level = EXPECTED_TOP_LEVEL | (
- CONTINUOUS_EXTRA_TOP_LEVEL if args.sha else set()
+ CONTINUOUS_EXTRA_TOP_LEVEL if (args.sha or args.expect_package_json)
+ else set()
)
archive_name = (
@@ -158,6 +165,14 @@ def main() -> int:
# The bundled catalog assets must be present and valid in the
# installed layout (the full packer gates: no small-fixture waiver).
catalog_facts = validate_bundled_catalog_dir(scratch)
+ if args.expect_package_json is not None:
+ stamped = json.loads((scratch / "package.json").read_text(encoding="utf-8"))
+ if stamped.get("version") != args.expect_package_json:
+ fail(f"package.json version is {stamped.get('version')!r}, "
+ f"expected the restamped release version "
+ f"{args.expect_package_json!r}")
+ if not str(stamped.get("commit", "")).strip():
+ fail("the restamped package.json lost its commit provenance")
binary = scratch / "prime-agent"
if not os.access(binary, os.X_OK):
fail("staged prime-agent is not executable")
@@ -174,6 +189,8 @@ def main() -> int:
expected_version = (
f"{args.version}-{CONTINUOUS_SUFFIX}.{args.sha}" if args.sha else args.version
)
+ if args.expect_package_json is not None:
+ expected_version = args.expect_package_json
if version_out != expected_version:
fail(
f"staged prime-agent reports {version_out!r}, "
diff --git a/scripts/test_ci_pr_crates.py b/scripts/test_ci_pr_crates.py
new file mode 100644
index 0000000000..a9d490a3a3
--- /dev/null
+++ b/scripts/test_ci_pr_crates.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Test battery for the PR-files-to-crates mapping (ci.yml's changes job).
+
+The contracts, pinned on real pulls/files-API shapes (the `filename` +
+`previous_filename` rows — the API's own field names, so a field-name
+regression fails safe instead of silently narrowing to nothing):
+ - a crate-only PR maps to those crates and their transitive dependents;
+ - ANY non-crate path - and a rename's removed side too - fails safe to
+ the full selection;
+ - an undecidable list (empty input, an empty filename row, the API's
+ 3000-file ceiling) fails safe to the full selection, never to an empty
+ selection: a narrowed wave that runs nothing would masquerade as green.
+
+Run: python3 scripts/test_ci_pr_crates.py (or: make shard-gates)
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+import ci_pr_crates # noqa: E402 (the tool under test)
+
+MAPPER = SCRIPTS_DIR / "ci_pr_crates.py"
+
+
+def run_mapper(rows: list[str]) -> tuple[int, str, dict[str, str]]:
+ output = Path(tempfile.mkdtemp()) / "github-output"
+ env = dict(os.environ)
+ env["GITHUB_OUTPUT"] = str(output)
+ listing = Path(tempfile.mkdtemp()) / "pr-files.tsv"
+ listing.write_text("\n".join(rows) + "\n", encoding="utf-8")
+ result = subprocess.run([sys.executable, str(MAPPER), str(listing)],
+ capture_output=True, text=True, env=env, check=False)
+ outputs = {}
+ for line in output.read_text().splitlines():
+ key, _, value = line.partition("=")
+ outputs[key] = value
+ return result.returncode, result.stdout, outputs
+
+
+class TheMappingTestCase(unittest.TestCase):
+
+ def test_a_crate_only_pr_narrows_to_those_crates(self):
+ rc, out, outputs = run_mapper([
+ "crates/pa-daemon/src/main.rs",
+ "crates/pa-daemon/src/roster.rs",
+ "crates/pa-tui/src/view/runs.rs",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "pa-cli,pa-daemon,pa-tui")
+ self.assertIn("crates (3: pa-cli, pa-daemon, pa-tui)", outputs["scope"])
+
+ def test_any_non_crate_path_fails_safe_to_the_full_selection(self):
+ rc, out, outputs = run_mapper([
+ "crates/pa-core/src/lib.rs",
+ ".github/workflows/ci.yml",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertEqual(outputs["scope"], "all")
+ self.assertIn("non-crate path .github/workflows/ci.yml", out)
+
+ def test_a_renames_removed_side_counts_too(self):
+ # A rename from a non-crate path INTO a crate: the PR removed the
+ # old path, so the full selection runs.
+ rc, out, outputs = run_mapper([
+ "crates/pa-cli/src/new.rs\tdocs/old-location.md",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertIn("non-crate path docs/old-location.md", out)
+
+ def test_a_rename_within_crates_narrows_to_both_crates(self):
+ rc, out, outputs = run_mapper([
+ "crates/pa-tui/src/editor/selection.rs\tcrates/pa-tui/src/selection.rs",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "pa-cli,pa-tui")
+
+ def test_pa_types_only_runs_every_crate(self):
+ rc, out, outputs = run_mapper(["crates/pa-types/src/lib.rs"])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], ",".join(sorted(ci_pr_crates.WORKSPACE_DEPS)))
+
+ def test_pa_tui_only_runs_pa_cli_too(self):
+ rc, out, outputs = run_mapper(["crates/pa-tui/src/lib.rs"])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "pa-cli,pa-tui")
+
+ def test_pa_telemetry_only_runs_all_dependents(self):
+ rc, out, outputs = run_mapper(["crates/pa-telemetry/src/lib.rs"])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"],
+ "pa-cli,pa-core,pa-daemon,pa-telemetry,pa-tui")
+
+ def test_an_unknown_crate_fails_safe(self):
+ rc, out, outputs = run_mapper(["crates/pa-future/src/lib.rs"])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertIn("unknown crate pa-future", out)
+
+ def test_an_empty_filename_row_fails_safe(self):
+ # The field-name regression the reviewer demanded pinned: a row whose
+ # filename column is empty (e.g. reading `.path` from the files API,
+ # whose field is `filename`) must never narrow the wave to nothing -
+ # the run fails safe to the full selection instead.
+ rc, out, outputs = run_mapper([
+ "\tcrates/pa-core/src/lib.rs",
+ "crates/pa-daemon/src/lib.rs",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertEqual(outputs["scope"], "all")
+ self.assertIn("empty filename", out)
+
+ def test_a_null_field_read_also_fails_safe(self):
+ # jq renders a missing field as the string "null": not a crate path,
+ # so the full selection runs (nothing narrows on a broken column).
+ rc, out, outputs = run_mapper([
+ "null\tdocs/removed.md",
+ "crates/pa-core/src/lib.rs",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+
+ def test_an_empty_list_fails_safe_to_the_full_selection(self):
+ rc, out, outputs = run_mapper([])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertEqual(outputs["scope"], "all")
+
+ def test_a_whitespace_prefixed_path_is_a_non_crate_path(self):
+ # The fields are read raw: a path with a leading space (or any
+ # non-crate prefix) must never be stripped into a crate mapping.
+ rc, out, outputs = run_mapper([
+ " crates/pa-core/src/lib.rs",
+ "crates/pa-tui/src/lib.rs",
+ ])
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertIn("non-crate path crates/pa-core/src/lib.rs", out)
+
+ def test_the_files_api_ceiling_fails_safe(self):
+ rows = ["crates/pa-core/src/lib.rs"] * ci_pr_crates.FILES_API_CEILING
+ rc, out, outputs = run_mapper(rows)
+ self.assertEqual(rc, 0, out)
+ self.assertEqual(outputs["crates"], "")
+ self.assertIn("ceiling", out)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/scripts/test_ci_test_shard.py b/scripts/test_ci_test_shard.py
new file mode 100644
index 0000000000..e1fe8aa1cc
--- /dev/null
+++ b/scripts/test_ci_test_shard.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+"""Test battery for the CI shard tooling (scripts/ci_test_shard.py +
+scripts/ci_test_shard_summary.py): the PR smoke's narrowed selection and
+the summary's scope-aware partition audit.
+
+The contracts, pinned on the REAL workspace enumeration (cargo metadata
+only - no builds):
+ - the stable crc32 assignment is untouched by --crates: a narrowed run's
+ unit lands in the same shard the full run assigns it;
+ - the full selection is the default (push waves) and audits green;
+ - a crate selection narrows the universe; the union of the shards' runs
+ must cover EXACTLY the selection - a narrowed run can never
+ masquerade as a full one, and coverage can never silently regress;
+ - a mixed-scope wave (one shard claiming a different selection) fails
+ the audit instead of auditing a chimera;
+ - a shard that skips one of its selected units fails;
+ - a unit that runs outside the selection fails;
+ - --print-selection reports the selection without running anything and
+ emits the units count the wave's conditional bins build reads.
+
+Run: python3 scripts/test_ci_test_shard.py (or: make shard-gates)
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parent
+REPO = SCRIPTS_DIR.parent
+
+TOTAL = 8
+
+
+def load_module(name: str, path: Path):
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+SHARD = load_module("ci_test_shard", SCRIPTS_DIR / "ci_test_shard.py")
+SUMMARY = load_module("ci_test_shard_summary", SCRIPTS_DIR / "ci_test_shard_summary.py")
+
+
+def run_summary(directory: Path):
+ return subprocess.run(
+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard_summary.py"),
+ "--total", str(TOTAL), "--dir", str(directory)],
+ capture_output=True, text=True, check=False)
+
+
+class EnumerationTestCase(unittest.TestCase):
+ """The stable assignment + the narrowed selection."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.units = SHARD.enumerate_units()
+ cls.all_ids = [u["id"] for u in cls.units]
+ cls.packages = {u["package"] for u in cls.units}
+
+ def write_wave(self, directory: Path, crates, results_by_shard=None,
+ scope_by_shard=None):
+ selection = SHARD.selected_units(self.units, crates)
+ for shard in range(1, TOTAL + 1):
+ mine = [u for u in selection
+ if SHARD.shard_of(u["id"], TOTAL) == shard - 1]
+ results = results_by_shard(shard, mine) if results_by_shard \
+ else [dict(id=u["id"], package=u["package"], kind=u["kind"],
+ name=u["name"], rc=0, seconds=1.0,
+ failed_tests=[]) for u in mine]
+ SHARD.write_manifest(
+ directory / f"shard-manifest-{shard}.json", shard, TOTAL,
+ self.all_ids, results, scope_by_shard(shard) if scope_by_shard else crates)
+ return selection
+
+ def test_the_assignment_is_stable_under_the_selection(self):
+ """The narrowed universe never moves a unit between shards: crc32
+ mod 8 is computed on the unit id, not the selection."""
+ for unit in self.units:
+ shard = SHARD.shard_of(unit["id"], TOTAL)
+ self.assertEqual(unit["id"],
+ [u["id"] for u in self.units
+ if SHARD.shard_of(u["id"], TOTAL) == shard
+ and u["id"] == unit["id"]][0])
+
+ def test_crate_selection_covers_exactly_that_crates_units(self):
+ crate = sorted(p for p in self.packages if p != "pa-agent")[0]
+ selection = SHARD.selected_units(self.units, [crate])
+ self.assertTrue(selection, "the crate selection is never empty")
+ self.assertTrue(all(u["package"] == crate for u in selection))
+ full = SHARD.selected_units(self.units, None)
+ self.assertLess(len(selection), len(full))
+ self.assertTrue(set(u["id"] for u in selection) <= set(u["id"] for u in full))
+
+
+class AuditTestCase(unittest.TestCase):
+ """The summary's scope-aware partition audit."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.units = SHARD.enumerate_units()
+ cls.all_ids = [u["id"] for u in cls.units]
+ # The audit-failure cases need a crate whose units spread across
+ # several shards: the largest unit count guarantees it.
+ counts = {}
+ for unit in cls.units:
+ counts[unit["package"]] = counts.get(unit["package"], 0) + 1
+ cls.crate = max((p for p in counts if p != "pa-agent"),
+ key=lambda p: counts[p])
+ cls.selection = [u["id"] for u in cls.units
+ if u["package"] == cls.crate]
+
+ def green_wave(self, directory: Path, crates, break_shard=None,
+ drop_unit=False, add_outside=False, scope_by_shard=None):
+ selection = [u["id"] for u in self.units
+ if crates is None or u["package"] in set(crates)]
+ for shard in range(1, TOTAL + 1):
+ mine = [i for i in selection if SHARD.shard_of(i, TOTAL) == shard - 1]
+ results = [dict(id=i, rc=0, seconds=1.0, failed_tests=[]) for i in mine]
+ if break_shard == shard and drop_unit and results:
+ results = results[:-1]
+ if break_shard == shard and add_outside:
+ outside = [i for i in self.all_ids
+ if i not in set(selection)
+ and SHARD.shard_of(i, TOTAL) == shard - 1]
+ if outside:
+ results.append(dict(id=outside[0], rc=0, seconds=1.0,
+ failed_tests=[]))
+ use_crates = scope_by_shard(shard) if scope_by_shard else crates
+ SHARD.write_manifest(
+ directory / f"shard-manifest-{shard}.json", shard, TOTAL,
+ self.all_ids, results, use_crates)
+
+ def test_a_full_wave_audits_green(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ directory = Path(tmp)
+ self.green_wave(directory, None)
+ result = run_summary(directory)
+ self.assertEqual(result.returncode, 0, result.stdout)
+ self.assertIn("full selection", result.stdout)
+
+ def test_a_narrowed_wave_audits_green_and_names_the_scope(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ directory = Path(tmp)
+ self.green_wave(directory, [self.crate])
+ result = run_summary(directory)
+ self.assertEqual(result.returncode, 0, result.stdout)
+ self.assertIn(f"crate selection: {self.crate}", result.stdout)
+ self.assertIn(f"{len(self.selection)} of {len(self.all_ids)} units selected",
+ result.stdout)
+
+ def test_a_mixed_scope_wave_fails_the_audit(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ directory = Path(tmp)
+ self.green_wave(directory, [self.crate],
+ scope_by_shard=lambda shard: None if shard == 5 else [self.crate])
+ result = run_summary(directory)
+ self.assertEqual(result.returncode, 1, result.stdout)
+ self.assertIn("different selection scopes", result.stdout)
+
+ def test_a_shard_that_skips_a_selected_unit_fails(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ directory = Path(tmp)
+ self.green_wave(directory, [self.crate], break_shard=4, drop_unit=True)
+ result = run_summary(directory)
+ self.assertEqual(result.returncode, 1, result.stdout)
+ self.assertIn("never reported", result.stdout)
+
+ def test_a_unit_that_runs_outside_the_selection_fails(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ directory = Path(tmp)
+ self.green_wave(directory, [self.crate], break_shard=2, add_outside=True)
+ result = run_summary(directory)
+ self.assertEqual(result.returncode, 1, result.stdout)
+ self.assertIn("outside the selection ran", result.stdout)
+
+ def test_a_selected_unit_no_shard_ran_fails(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ directory = Path(tmp)
+ self.green_wave(directory, [self.crate])
+ # Drop the unit from the run AND from the shard's recorded
+ # selection reality: rewrite one manifest with the unit missing
+ # from its units and its complete flag falsified.
+ shard = next(SHARD.shard_of(i, TOTAL) + 1 for i in self.selection)
+ path = directory / f"shard-manifest-{shard}.json"
+ manifest = json.loads(path.read_text())
+ if manifest["units"]:
+ manifest["units"] = manifest["units"][:-1]
+ manifest["complete"] = False
+ path.write_text(json.dumps(manifest, indent=1) + "\n")
+ result = run_summary(directory)
+ self.assertEqual(result.returncode, 1, result.stdout)
+
+
+class SelectionReportingTestCase(unittest.TestCase):
+ """--print-selection resolves without running and emits the count."""
+
+ def test_print_selection_reports_and_counts(self):
+ env = dict(os.environ)
+ output = Path(tempfile.mkdtemp()) / "github-output"
+ env["GITHUB_OUTPUT"] = str(output)
+ result = subprocess.run(
+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard.py"),
+ "--shard", "3", "--total", str(TOTAL), "--crates", "pa-daemon",
+ "--print-selection"],
+ capture_output=True, text=True, env=env, cwd=REPO, check=False)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("SELECTED=", result.stdout)
+ self.assertIn("crate selection: pa-daemon", result.stdout)
+ count = int(next(line for line in result.stdout.splitlines()
+ if line.startswith("SELECTED=")).split("=")[1])
+ self.assertGreater(count, 0)
+ self.assertIn(f"units={count}", output.read_text())
+
+ def test_an_empty_selection_reports_zero_and_skips_nothing(self):
+ """A crate whose units all hash to OTHER shards leaves a leg with
+ nothing to run: the bins build reads units=0 and skips."""
+ env = dict(os.environ)
+ output = Path(tempfile.mkdtemp()) / "github-output"
+ env["GITHUB_OUTPUT"] = str(output)
+ # pa-ai's two units both hash to shard 8: every other leg selects zero.
+ result = subprocess.run(
+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard.py"),
+ "--shard", "3", "--total", str(TOTAL), "--crates", "pa-ai",
+ "--print-selection"],
+ capture_output=True, text=True, env=env, cwd=REPO, check=False)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("SELECTED=0", result.stdout)
+ self.assertIn("units=0", output.read_text())
+
+ def test_an_unknown_package_is_refused_fail_closed(self):
+ """A --crates name the workspace does not build is a mapping bug:
+ refuse it (a silent zero would masquerade as a green wave)."""
+ result = subprocess.run(
+ [sys.executable, str(SCRIPTS_DIR / "ci_test_shard.py"),
+ "--shard", "3", "--total", str(TOTAL), "--crates", "pa-nonexistent",
+ "--print-selection"],
+ capture_output=True, text=True, cwd=REPO, check=False)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("does not build", result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)