Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Recover activation for release-train snapshot runs
rcourtman54fd5f3aa234
diff --git a/.github/workflows/recover-release-activation.yml b/.github/workflows/recover-release-activation.ymlindex 5a4274f3dd..8f5c774199 100644--- a/.github/workflows/recover-release-activation.yml+++ b/.github/workflows/recover-release-activation.yml@@ -24,6 +24,7 @@ permissions:actions: writecontents: writepackages: read+ pull-requests: readjobs:recover_activation:@@ -35,6 +36,8 @@ jobs:uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:persist-credentials: false+ # Snapshot provenance checks that the merged release PR is in main's history.+ fetch-depth: 0- name: Qualify the quarantined source runid: qualify@@ -44,6 +47,11 @@ jobs:SOURCE_RELEASE_RUN_ID: ${{ inputs.source_release_run_id }}run: |set -euo pipefail+ # Snapshot provenance below must be checked by main's own code.+ if [[ "${GITHUB_REF}" != refs/heads/main ]]; then+ echo "::error::Activation recovery must be dispatched on main, not ${GITHUB_REF}."+ exit 1+ fiif [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-((rc|alpha|beta)\.[0-9]+))?$ ]]; thenecho "::error::Invalid recovery tag ${TAG}."exit 1@@ -60,19 +68,52 @@ jobs:trap cleanup EXITgh api "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RELEASE_RUN_ID}" > "${source_run}"+ # The release train dispatches on a release-candidate/ or post-merge+ # release-dispatch/ snapshot ref rather than main. Such a run is+ # admitted only after the snapshot check below proves it ran at the+ # exact head of a merged release PR.jq -e \--arg repository "${GITHUB_REPOSITORY}" \'.event == "workflow_dispatch" and.path == ".github/workflows/create-release.yml" and- .head_branch == "main" and+ (.head_branch == "main" or+ (.head_branch | test("^release-(candidate|dispatch)/[0-9A-Za-z._-]+$"))) and.head_repository.full_name == $repository and.status == "completed" and.conclusion == "failure" and(.head_sha | test("^[0-9a-f]{40}$"))' \"${source_run}" >/dev/null || {- echo "::error::Run ${SOURCE_RELEASE_RUN_ID} is not a completed failed mainline create-release run."+ echo "::error::Run ${SOURCE_RELEASE_RUN_ID} is not a completed failed create-release run from main or a release snapshot."exit 1}+ source_branch=$(jq -r '.head_branch' "${source_run}")+ if [[ "${source_branch}" != main ]]; then+ # The source run's own prepare job ran that branch's code, so it+ # proves nothing here. Re-check provenance with main's snapshot+ # check: the run's commit must be the merged head of its release PR.+ candidate_branch="${source_branch}"+ if [[ "${source_branch}" == release-dispatch/* ]]; then+ candidate_branch="release-candidate/${source_branch#release-dispatch/}"+ fi+ run_sha=$(jq -r '.head_sha' "${source_run}")+ snapshot_prs=$(gh api \+ "repos/${GITHUB_REPOSITORY}/pulls?state=closed&base=main&head=${GITHUB_REPOSITORY_OWNER}:${candidate_branch}&per_page=100" \+ | jq -r --arg ref "${candidate_branch}" --arg sha "${run_sha}" \+ '[.[] | select(.merged_at != null and .head.ref == $ref and .head.sha == $sha) | .number] | @tsv')+ if [[ ! "${snapshot_prs}" =~ ^[1-9][0-9]*$ ]]; then+ echo "::error::Run ${SOURCE_RELEASE_RUN_ID} on ${source_branch} is not the merged head of exactly one ${candidate_branch} release PR."+ exit 1+ fi+ snapshot_output=$(mktemp)+ GITHUB_REF="refs/heads/${source_branch}" GITHUB_SHA="${run_sha}" GITHUB_WORKFLOW_SHA="${run_sha}" \+ RELEASE_SOURCE_BRANCH=main RELEASE_PULL_REQUEST="${snapshot_prs}" GITHUB_OUTPUT="${snapshot_output}" \+ python3 scripts/release_control/release_snapshot.py || {+ rm -f "${snapshot_output}"+ echo "::error::Run ${SOURCE_RELEASE_RUN_ID} failed release snapshot provenance."+ exit 1+ }+ rm -f "${snapshot_output}"+ figh api --paginate --slurp \"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RELEASE_RUN_ID}/jobs?per_page=100" \diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 0439dabb48..ca9fb4b75d 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -3544,7 +3544,15 @@ artifact-selection behaviour.depending on current-directory repository discovery.A failed activation after immutable readiness has passed may use theactivation-only recovery workflow. Recovery must accept only a completed- failed `create-release.yml` run whose failures are confined to activation,+ failed `create-release.yml` run whose failures are confined to activation.+ The run may come from `main` or from a release-train snapshot ref+ (`release-candidate/<packet>` or `release-dispatch/<packet>`). A snapshot run+ is admitted only when main's `release_snapshot.py` proves it ran at the+ merged head of exactly one `release-candidate/<packet>` release PR; the+ source run's own `prepare` job ran that ref's code and proves nothing.+ Recovery itself refuses to run unless dispatched on `main`, so that check is+ main's code.+ Recovery must alsorequire the successful `release_readiness` DAG join as the canonical proofthat every immutable gate succeeded when the source run has that job, andreject every failure outside the activation boundary. Runs made after thediff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.pyindex 0576d92a8d..b2c7f30009 100644--- a/scripts/release_control/release_promotion_policy_test.py+++ b/scripts/release_control/release_promotion_policy_test.py@@ -750,6 +750,82 @@ class ReleasePromotionPolicyTest(unittest.TestCase):self.assertFalse(qualifies(replaced(new_shape, "backend_tests", "failure")))self.assertFalse(qualifies(replaced(old_shape, "backend_tests", "failure")))+ def test_activation_recovery_admits_snapshot_runs_only_with_merged_provenance(self) -> None:+ # Execute the recovery's source-run admission. Release-train runs are+ # dispatched on release-candidate/ or release-dispatch/ snapshot refs,+ # and must prove, with main's snapshot check rather than their own+ # prepare job, that they ran at the merged head of their release PR.+ workflow = read(".github/workflows/recover-release-activation.yml")+ job = workflow_job_block(workflow, "recover_activation")+ # The provenance check is only main's if recovery itself runs on main.+ self.assertIn('if [[ "${GITHUB_REF}" != refs/heads/main ]]; then', job)+ self.assertIn(" pull-requests: read\n", workflow)+ self.assertIn("fetch-depth: 0", job)+ start = job.index('gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RELEASE_RUN_ID}" > "${source_run}"')+ start = job.index("\n", start) + 1+ end = job.index("gh api --paginate --slurp")+ script = "set -euo pipefail\n" + job[start:end] + "echo QUALIFIED\n"+ sha, merge = "a" * 40, "b" * 40++ def pull(number: int, ref: str, head_sha: str = sha, merged: bool = True) -> dict:+ return {"number": number, "state": "closed", "merged": merged,+ "merged_at": "2026-10-25T00:00:00Z" if merged else None,+ "merge_commit_sha": merge if merged else None,+ "head": {"ref": ref, "sha": head_sha, "repo": {"full_name": "rcourtman/Pulse"}},+ "base": {"ref": "main", "repo": {"full_name": "rcourtman/Pulse"}}}++ def qualifies(branch: str, pulls: list[dict], *, git_result: int = 0) -> bool:+ with tempfile.TemporaryDirectory() as temp:+ root = Path(temp)+ (root / "run.json").write_text(json.dumps({+ "event": "workflow_dispatch", "path": ".github/workflows/create-release.yml",+ "head_branch": branch, "head_sha": sha, "status": "completed", "conclusion": "failure",+ "head_repository": {"full_name": "rcourtman/Pulse"}}))+ (root / "pulls.json").write_text(json.dumps(pulls))+ stubs = root / "bin"+ stubs.mkdir()+ (stubs / "gh").write_text(+ "#!/bin/bash\n"+ "case \"$*\" in\n"+ f" *pulls\\?state=closed*) cat {root}/pulls.json ;;\n"+ " *pulls/*) python3 -c 'import json,sys; n=int(sys.argv[2].rsplit(\"/\",1)[1]);"+ " print(json.dumps(next(p for p in json.load(open(sys.argv[1])) if p[\"number\"]==n)))'"+ f" {root}/pulls.json \"${{@: -1}}\" ;;\n"+ " *) exit 1 ;;\n"+ "esac\n")+ (stubs / "git").write_text(f"#!/bin/bash\nexit {git_result}\n")+ for stub in stubs.iterdir():+ stub.chmod(0o755)+ result = subprocess.run(+ ["bash", "-c", script], cwd=REPO_ROOT, text=True, capture_output=True,+ env=os.environ | {"PATH": f"{stubs}:{os.environ['PATH']}", "source_run": str(root / "run.json"),+ "SOURCE_RELEASE_RUN_ID": "1", "GITHUB_REPOSITORY": "rcourtman/Pulse",+ "GITHUB_REPOSITORY_OWNER": "rcourtman"},+ )+ self.assertEqual(result.returncode == 0, "QUALIFIED" in result.stdout, result.stderr)+ return result.returncode == 0++ candidate = "release-candidate/v6.7.0-rc.1-20261101T000000Z"+ dispatch = "release-dispatch/v6.7.0-rc.1-20261101T000000Z"+ self.assertTrue(qualifies("main", []))+ self.assertTrue(qualifies(candidate, [pull(7, candidate)]))+ self.assertTrue(qualifies(dispatch, [pull(7, candidate)]))+ rejected = {+ "lane branch": ("maintainer/run-1", [pull(7, "maintainer/run-1")]),+ "no release PR": (dispatch, []),+ "another candidate": (dispatch, [pull(7, "release-candidate/v6.7.0-rc.2-20261102T000000Z")]),+ "dispatch-named PR head": (dispatch, [pull(7, dispatch)]),+ "unmerged PR": (candidate, [pull(7, candidate, merged=False)]),+ "different head commit": (dispatch, [pull(7, candidate, head_sha="c" * 40)]),+ "two matching PRs": (dispatch, [pull(7, candidate), pull(8, candidate)]),+ "malformed snapshot ref": ("release-dispatch/a/b", [pull(7, "release-candidate/a/b")]),+ }+ for name, (branch, pulls) in rejected.items():+ with self.subTest(rejected=name):+ self.assertFalse(qualifies(branch, pulls))+ with self.subTest(rejected="merge missing from main"):+ self.assertFalse(qualifies(dispatch, [pull(7, candidate)], git_result=1))+def test_commit_verdict_restates_the_candidate_predicate(self) -> None:# The verdict no longer reads a readiness join, so it must reject every# candidate failure itself. Run its result checks with each outcome.