Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Re-dispatch Pro promotion refused by a moved pulse-pro main
rcourtman2d498a35d2e6
diff --git a/.github/workflows/promote-private-pro-runtime.yml b/.github/workflows/promote-private-pro-runtime.ymlindex df895459c1..5534e295bc 100644--- a/.github/workflows/promote-private-pro-runtime.yml+++ b/.github/workflows/promote-private-pro-runtime.yml@@ -143,12 +143,12 @@ jobs:url="$(awk -F '\t' '{print $3}' <<<"${run_state}")"echo "${label}: status=${status} conclusion=${conclusion:-pending} ${url}"if [[ "${status}" == "completed" ]]; then- if [[ "${conclusion}" == "success" ]]; then- echo "[OK] ${label} completed successfully: ${url}"- return 0- fi- echo "::error::${label} failed with conclusion=${conclusion}: ${url}"- return 1+ # The caller judges the conclusion: a refused attempt it+ # re-dispatches must leave no failure annotation, because+ # reconciliation reads exactly one private run URL from them.+ completed_conclusion="${conclusion}"+ completed_url="${url}"+ return 0fisleep 5done@@ -157,39 +157,82 @@ jobs:return 1}+ # pulse-pro gives a run credentials only while that run is still its+ # current main (scripts/check_workflow_source.py). A pulse-pro merge+ # landing during the run's queue and validation window therefore+ # refuses the promotion before anything is mutated; on 24 October+ # 2026 that left v6.6.0-rc.6 off the RC broker until the hourly+ # reconcile re-ran convergence. Only that refusal is re-dispatched.+ source_check_refused_only() {+ local run_id="$1"+ local run_head current_head refused+ run_head="$(gh api "repos/rcourtman/pulse-pro/actions/runs/${run_id}" --jq '.head_sha')"+ current_head="$(gh api repos/rcourtman/pulse-pro/git/ref/heads/main --jq '.object.sha')"+ if [[ ! "${run_head}" =~ ^[0-9a-f]{40}$ ]] || \+ [[ ! "${current_head}" =~ ^[0-9a-f]{40}$ ]] || \+ [[ "${run_head}" == "${current_head}" ]]; then+ return 1+ fi+ refused="$(+ gh run view "${run_id}" \+ --repo rcourtman/pulse-pro \+ --json jobs \+ --jq '[.jobs[] | select(.conclusion == "failure") | [.steps[] | select(.conclusion == "failure") | .name] | unique] | length > 0 and all(. == ["Require current exact main source before credentials"])'+ )"+ [[ "${refused}" == "true" ]]+ }+allow_ga_publish=falseif [[ "${IS_PRERELEASE}" != "true" ]]; thenallow_ga_publish=truefi- echo "Dispatching live paid-runtime promotion for ${TAG} with R2 prefix ${R2_PREFIX}."- promote_dispatch="$(- jq -n \- --arg version "${VERSION}" \- --arg r2_prefix "${R2_PREFIX}" \- --arg allow_ga_prefix "${allow_ga_publish}" \- --arg pulse_lease_sha "${PULSE_LEASE_SHA}" \- --arg pulse_convergence_run_id "${PULSE_CONVERGENCE_RUN_ID}" \- --arg pulse_owner_asset_name "${PULSE_OWNER_ASSET_NAME}" \- --arg pulse_owner_asset_sha256 "${PULSE_OWNER_ASSET_SHA256}" \- '{- ref: "main",- return_run_details: true,- inputs: {- version: $version,- r2_prefix: $r2_prefix,- allow_ga_prefix: $allow_ga_prefix,- pulse_lease_sha: $pulse_lease_sha,- pulse_convergence_run_id: $pulse_convergence_run_id,- pulse_owner_asset_name: $pulse_owner_asset_name,- pulse_owner_asset_sha256: $pulse_owner_asset_sha256- }- }' | \- gh api \- --method POST \- -H "Accept: application/vnd.github+json" \- -H "X-GitHub-Api-Version: 2026-03-10" \- repos/rcourtman/pulse-pro/actions/workflows/promote-paid-runtime-release.yml/dispatches \- --input -- )"- promote_run_id="$(jq -r '.workflow_run_id // empty' <<<"${promote_dispatch}")"- wait_for_workflow rcourtman/pulse-pro "${promote_run_id}" "private Pro live promotion" 3600+ max_promotion_attempts=5+ promotion_deadline=$((SECONDS + 3600))+ for attempt in $(seq 1 "${max_promotion_attempts}"); do+ echo "Dispatching live paid-runtime promotion for ${TAG} with R2 prefix ${R2_PREFIX} (attempt ${attempt}/${max_promotion_attempts})."+ promote_dispatch="$(+ jq -n \+ --arg version "${VERSION}" \+ --arg r2_prefix "${R2_PREFIX}" \+ --arg allow_ga_prefix "${allow_ga_publish}" \+ --arg pulse_lease_sha "${PULSE_LEASE_SHA}" \+ --arg pulse_convergence_run_id "${PULSE_CONVERGENCE_RUN_ID}" \+ --arg pulse_owner_asset_name "${PULSE_OWNER_ASSET_NAME}" \+ --arg pulse_owner_asset_sha256 "${PULSE_OWNER_ASSET_SHA256}" \+ '{+ ref: "main",+ return_run_details: true,+ inputs: {+ version: $version,+ r2_prefix: $r2_prefix,+ allow_ga_prefix: $allow_ga_prefix,+ pulse_lease_sha: $pulse_lease_sha,+ pulse_convergence_run_id: $pulse_convergence_run_id,+ pulse_owner_asset_name: $pulse_owner_asset_name,+ pulse_owner_asset_sha256: $pulse_owner_asset_sha256+ }+ }' | \+ gh api \+ --method POST \+ -H "Accept: application/vnd.github+json" \+ -H "X-GitHub-Api-Version: 2026-03-10" \+ repos/rcourtman/pulse-pro/actions/workflows/promote-paid-runtime-release.yml/dispatches \+ --input -+ )"+ promote_run_id="$(jq -r '.workflow_run_id // empty' <<<"${promote_dispatch}")"+ completed_conclusion=""+ completed_url=""+ wait_for_workflow rcourtman/pulse-pro "${promote_run_id}" "private Pro live promotion" "$((promotion_deadline - SECONDS))"+ if [[ "${completed_conclusion}" == "success" ]]; then+ echo "[OK] private Pro live promotion completed successfully: ${completed_url}"+ exit 0+ fi+ if [[ "${completed_conclusion}" == "failure" ]] && \+ (( attempt < max_promotion_attempts )) && \+ source_check_refused_only "${promote_run_id}"; then+ echo "private Pro live promotion run ${promote_run_id} was refused only because pulse-pro main moved after dispatch; re-dispatching on the current main."+ continue+ fi+ echo "::error::private Pro live promotion failed with conclusion=${completed_conclusion}: ${completed_url}"+ exit 1+ donediff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 34d56af2f4..462cda664b 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -3374,6 +3374,16 @@ artifact-selection behaviour.A rebuilt packet from identical inputs is waste and lineage churn; anon-empty prefix that fails packet validation must fail the private buildinstead of being overwritten.+ The caller itself re-dispatches only one failure: a refusal by the mutator's+ current-main source check. That check runs before credentials, so nothing+ was mutated. Every failed job must have failed only that step, and+ `pulse-pro` main must have moved off the refused run's head. The caller then+ dispatches again on the new main, up to five attempts within one hour. Any+ other failed step, an unchanged head or a cancelled run stays terminal.+ Only the terminal failure is annotated, so reconciliation still reads+ exactly one private run URL. On 12, 17 and 24 October 2026, unrelated+ `pulse-pro` merges during the run's queue and validation window caused five+ such refusals, and each one failed convergence.A successfully uploaded `release-activation.json` asset is the irreversiblerelease commit, not the earlier `draft=false` PATCH by itself. Before thatmarker, `activate_release` must verify the draft identity and every requireddiff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.pyindex 53a53a3842..e4298e4344 100644--- a/scripts/release_control/release_promotion_policy_test.py+++ b/scripts/release_control/release_promotion_policy_test.py@@ -1395,6 +1395,124 @@ class ReleasePromotionPolicyTest(unittest.TestCase):self.assertNotIn("gh run list", job)self.assertNotIn("started_at", job)+ def test_private_pro_promotion_redispatches_only_a_moved_main_source_refusal(self) -> None:+ """Run the real dispatch loop against doubles: only a pre-credential refusal caused by a+ pulse-pro main that moved after dispatch is re-dispatched, and only the terminal failure+ leaves the single private-run annotation that reconciliation reads."""+ workflow = yaml.safe_load(read(".github/workflows/promote-private-pro-runtime.yml"))+ step = workflow["jobs"]["promote"]["steps"][0]["run"]+ script = step[step.index("wait_for_workflow() {"):]+ source_step = "Require current exact main source before credentials"+ fake_gh = r'''#!/usr/bin/env python3+import json, os, subprocess, sys+fix = os.environ["FIX"]+args = sys.argv[1:]+expression = args[args.index("--jq") + 1] if "--jq" in args else None+with open(os.path.join(fix, "state.json")) as handle:+ state = json.load(handle)++def emit(value):+ text = json.dumps(value)+ if expression is not None:+ text = subprocess.run(["jq", "-r", expression], input=text, text=True,+ capture_output=True, check=True).stdout+ sys.stdout.write(text if text.endswith("\n") else text + "\n")++def attempt(run_id):+ return state["attempts"][min(int(run_id) - 100, len(state["attempts"]) - 1)]++if args[0] == "api" and "--method" in args:+ sys.stdin.read()+ run_id = 100 + len(state["dispatched"])+ state["dispatched"].append(run_id)+ with open(os.path.join(fix, "state.json"), "w") as handle:+ json.dump(state, handle)+ emit({"workflow_run_id": run_id})+elif args[0] == "api" and args[1].endswith("/git/ref/heads/main"):+ emit({"object": {"sha": state["main"]}})+elif args[0] == "api" and "/actions/runs/" in args[1]:+ emit({"head_sha": attempt(args[1].rsplit("/", 1)[1])["head"]})+elif args[:2] == ["run", "view"] and "jobs" in args:+ run = attempt(args[2])+ jobs = [{"name": "credential_containment", "conclusion": "success",+ "steps": [{"name": "Checkout pulse-pro", "conclusion": "success"}]}]+ if run["failed_step"]:+ jobs.append({"name": "promote", "conclusion": "failure", "steps": [+ {"name": "Checkout pulse-pro", "conclusion": "success"},+ {"name": run["failed_step"], "conclusion": "failure"}]})+ emit({"jobs": jobs})+elif args[:2] == ["run", "view"]:+ emit({"status": "completed", "conclusion": attempt(args[2])["conclusion"],+ "url": f"https://github.com/rcourtman/pulse-pro/actions/runs/{args[2]}"})+else:+ sys.exit(f"unexpected gh call: {args}")+'''+ old_main, new_main = "a" * 40, "b" * 40++ def run(main: str, attempts: list[dict]) -> tuple[subprocess.CompletedProcess, list[int]]:+ with tempfile.TemporaryDirectory() as directory:+ fix = Path(directory)+ (fix / "bin").mkdir()+ for name, body in (("gh", fake_gh), ("sleep", "#!/bin/bash\n")):+ (fix / "bin" / name).write_text(body)+ (fix / "bin" / name).chmod(0o755)+ (fix / "state.json").write_text(+ json.dumps({"main": main, "attempts": attempts, "dispatched": []})+ )+ env = {**os.environ, "FIX": str(fix), "TAG": "v6.6.0-rc.6", "VERSION": "6.6.0-rc.6",+ "IS_PRERELEASE": "true", "R2_PREFIX": "prefix", "PULSE_LEASE_SHA": "c" * 40,+ "PULSE_CONVERGENCE_RUN_ID": "7", "PULSE_OWNER_ASSET_NAME": "owner.json",+ "PULSE_OWNER_ASSET_SHA256": "d" * 64,+ "PATH": f"{fix / 'bin'}{os.pathsep}{os.environ['PATH']}"}+ result = subprocess.run(+ ["bash", "--noprofile", "--norc", "-eu", "-o", "pipefail", "-c", script],+ cwd=REPO_ROOT, env=env, text=True, capture_output=True, timeout=60,+ )+ dispatched = json.loads((fix / "state.json").read_text())["dispatched"]+ return result, dispatched++ refused = {"conclusion": "failure", "head": old_main, "failed_step": source_step}+ promoted = {"conclusion": "success", "head": new_main, "failed_step": ""}++ # A refusal because main moved after dispatch is re-dispatched and leaves no annotation.+ result, dispatched = run(new_main, [refused, promoted])+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(dispatched, [100, 101])+ self.assertNotIn("::error::", result.stdout)+ self.assertIn("re-dispatching on the current main", result.stdout)++ # The same refusal while main is unchanged is a source-read fault, not a race: terminal.+ result, dispatched = run(old_main, [refused])+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(dispatched, [100])++ # A failure past the source check may have used credentials: never re-dispatched.+ mutated = {"conclusion": "failure", "head": old_main,+ "failed_step": "Download signed packet and promote live broker"}+ result, dispatched = run(new_main, [mutated])+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(dispatched, [100])+ self.assertEqual(+ re.findall(r"::error::.*", result.stdout),+ ["::error::private Pro live promotion failed with conclusion=failure: "+ "https://github.com/rcourtman/pulse-pro/actions/runs/100"],+ )++ # A cancelled run is not a refusal either.+ result, dispatched = run(new_main, [{**refused, "conclusion": "cancelled"}])+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(dispatched, [100])++ # Retries are bounded, and only the last attempt's URL is annotated.+ result, dispatched = run(new_main, [refused])+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(dispatched, [100, 101, 102, 103, 104])+ self.assertEqual(+ re.findall(r"https://github\.com/rcourtman/pulse-pro/actions/runs/\d+",+ "\n".join(re.findall(r"::error::.*", result.stdout))),+ ["https://github.com/rcourtman/pulse-pro/actions/runs/104"],+ )+def test_release_promotion_policy_requires_live_rc_and_v5_policy(self) -> None:content = read("docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md")self.assertIn("`beta.N` is the normal user-testing stage", content)