Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Re-dispatch Pro promotion refused by a moved pulse-pro main

rcourtman2d498a35d2e6
diff --git a/.github/workflows/promote-private-pro-runtime.yml b/.github/workflows/promote-private-pro-runtime.yml
index df895459c1..5534e295bc 100644
--- a/.github/workflows/promote-private-pro-runtime.yml
+++ b/.github/workflows/promote-private-pro-runtime.yml
@@ -143,12 +143,12 @@ jobs:
url="$(awk -F '\t' '{print $3}' <<<"${run_state}")"
echo "${label}: status=${status} conclusion=${conclusion:-pending} ${url}"
if [[ "${status}" == "completed" ]]; then
- if [[ "${conclusion}" == "success" ]]; then
- echo "[OK] ${label} completed successfully: ${url}"
- return 0
- fi
- echo "::error::${label} failed with conclusion=${conclusion}: ${url}"
- return 1
+ # The caller judges the conclusion: a refused attempt it
+ # re-dispatches must leave no failure annotation, because
+ # reconciliation reads exactly one private run URL from them.
+ completed_conclusion="${conclusion}"
+ completed_url="${url}"
+ return 0
fi
sleep 5
done
@@ -157,39 +157,82 @@ jobs:
return 1
}
+ # pulse-pro gives a run credentials only while that run is still its
+ # current main (scripts/check_workflow_source.py). A pulse-pro merge
+ # landing during the run's queue and validation window therefore
+ # refuses the promotion before anything is mutated; on 24 October
+ # 2026 that left v6.6.0-rc.6 off the RC broker until the hourly
+ # reconcile re-ran convergence. Only that refusal is re-dispatched.
+ source_check_refused_only() {
+ local run_id="$1"
+ local run_head current_head refused
+ run_head="$(gh api "repos/rcourtman/pulse-pro/actions/runs/${run_id}" --jq '.head_sha')"
+ current_head="$(gh api repos/rcourtman/pulse-pro/git/ref/heads/main --jq '.object.sha')"
+ if [[ ! "${run_head}" =~ ^[0-9a-f]{40}$ ]] || \
+ [[ ! "${current_head}" =~ ^[0-9a-f]{40}$ ]] || \
+ [[ "${run_head}" == "${current_head}" ]]; then
+ return 1
+ fi
+ refused="$(
+ gh run view "${run_id}" \
+ --repo rcourtman/pulse-pro \
+ --json jobs \
+ --jq '[.jobs[] | select(.conclusion == "failure") | [.steps[] | select(.conclusion == "failure") | .name] | unique] | length > 0 and all(. == ["Require current exact main source before credentials"])'
+ )"
+ [[ "${refused}" == "true" ]]
+ }
+
allow_ga_publish=false
if [[ "${IS_PRERELEASE}" != "true" ]]; then
allow_ga_publish=true
fi
- echo "Dispatching live paid-runtime promotion for ${TAG} with R2 prefix ${R2_PREFIX}."
- promote_dispatch="$(
- jq -n \
- --arg version "${VERSION}" \
- --arg r2_prefix "${R2_PREFIX}" \
- --arg allow_ga_prefix "${allow_ga_publish}" \
- --arg pulse_lease_sha "${PULSE_LEASE_SHA}" \
- --arg pulse_convergence_run_id "${PULSE_CONVERGENCE_RUN_ID}" \
- --arg pulse_owner_asset_name "${PULSE_OWNER_ASSET_NAME}" \
- --arg pulse_owner_asset_sha256 "${PULSE_OWNER_ASSET_SHA256}" \
- '{
- ref: "main",
- return_run_details: true,
- inputs: {
- version: $version,
- r2_prefix: $r2_prefix,
- allow_ga_prefix: $allow_ga_prefix,
- pulse_lease_sha: $pulse_lease_sha,
- pulse_convergence_run_id: $pulse_convergence_run_id,
- pulse_owner_asset_name: $pulse_owner_asset_name,
- pulse_owner_asset_sha256: $pulse_owner_asset_sha256
- }
- }' | \
- gh api \
- --method POST \
- -H "Accept: application/vnd.github+json" \
- -H "X-GitHub-Api-Version: 2026-03-10" \
- repos/rcourtman/pulse-pro/actions/workflows/promote-paid-runtime-release.yml/dispatches \
- --input -
- )"
- promote_run_id="$(jq -r '.workflow_run_id // empty' <<<"${promote_dispatch}")"
- wait_for_workflow rcourtman/pulse-pro "${promote_run_id}" "private Pro live promotion" 3600
+ max_promotion_attempts=5
+ promotion_deadline=$((SECONDS + 3600))
+ for attempt in $(seq 1 "${max_promotion_attempts}"); do
+ echo "Dispatching live paid-runtime promotion for ${TAG} with R2 prefix ${R2_PREFIX} (attempt ${attempt}/${max_promotion_attempts})."
+ promote_dispatch="$(
+ jq -n \
+ --arg version "${VERSION}" \
+ --arg r2_prefix "${R2_PREFIX}" \
+ --arg allow_ga_prefix "${allow_ga_publish}" \
+ --arg pulse_lease_sha "${PULSE_LEASE_SHA}" \
+ --arg pulse_convergence_run_id "${PULSE_CONVERGENCE_RUN_ID}" \
+ --arg pulse_owner_asset_name "${PULSE_OWNER_ASSET_NAME}" \
+ --arg pulse_owner_asset_sha256 "${PULSE_OWNER_ASSET_SHA256}" \
+ '{
+ ref: "main",
+ return_run_details: true,
+ inputs: {
+ version: $version,
+ r2_prefix: $r2_prefix,
+ allow_ga_prefix: $allow_ga_prefix,
+ pulse_lease_sha: $pulse_lease_sha,
+ pulse_convergence_run_id: $pulse_convergence_run_id,
+ pulse_owner_asset_name: $pulse_owner_asset_name,
+ pulse_owner_asset_sha256: $pulse_owner_asset_sha256
+ }
+ }' | \
+ gh api \
+ --method POST \
+ -H "Accept: application/vnd.github+json" \
+ -H "X-GitHub-Api-Version: 2026-03-10" \
+ repos/rcourtman/pulse-pro/actions/workflows/promote-paid-runtime-release.yml/dispatches \
+ --input -
+ )"
+ promote_run_id="$(jq -r '.workflow_run_id // empty' <<<"${promote_dispatch}")"
+ completed_conclusion=""
+ completed_url=""
+ wait_for_workflow rcourtman/pulse-pro "${promote_run_id}" "private Pro live promotion" "$((promotion_deadline - SECONDS))"
+ if [[ "${completed_conclusion}" == "success" ]]; then
+ echo "[OK] private Pro live promotion completed successfully: ${completed_url}"
+ exit 0
+ fi
+ if [[ "${completed_conclusion}" == "failure" ]] && \
+ (( attempt < max_promotion_attempts )) && \
+ source_check_refused_only "${promote_run_id}"; then
+ echo "private Pro live promotion run ${promote_run_id} was refused only because pulse-pro main moved after dispatch; re-dispatching on the current main."
+ continue
+ fi
+ echo "::error::private Pro live promotion failed with conclusion=${completed_conclusion}: ${completed_url}"
+ exit 1
+ done
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 34d56af2f4..462cda664b 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -3374,6 +3374,16 @@ artifact-selection behaviour.
A rebuilt packet from identical inputs is waste and lineage churn; a
non-empty prefix that fails packet validation must fail the private build
instead of being overwritten.
+ The caller itself re-dispatches only one failure: a refusal by the mutator's
+ current-main source check. That check runs before credentials, so nothing
+ was mutated. Every failed job must have failed only that step, and
+ `pulse-pro` main must have moved off the refused run's head. The caller then
+ dispatches again on the new main, up to five attempts within one hour. Any
+ other failed step, an unchanged head or a cancelled run stays terminal.
+ Only the terminal failure is annotated, so reconciliation still reads
+ exactly one private run URL. On 12, 17 and 24 October 2026, unrelated
+ `pulse-pro` merges during the run's queue and validation window caused five
+ such refusals, and each one failed convergence.
A successfully uploaded `release-activation.json` asset is the irreversible
release commit, not the earlier `draft=false` PATCH by itself. Before that
marker, `activate_release` must verify the draft identity and every required
diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py
index 53a53a3842..e4298e4344 100644
--- a/scripts/release_control/release_promotion_policy_test.py
+++ b/scripts/release_control/release_promotion_policy_test.py
@@ -1395,6 +1395,124 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertNotIn("gh run list", job)
self.assertNotIn("started_at", job)
+ def test_private_pro_promotion_redispatches_only_a_moved_main_source_refusal(self) -> None:
+ """Run the real dispatch loop against doubles: only a pre-credential refusal caused by a
+ pulse-pro main that moved after dispatch is re-dispatched, and only the terminal failure
+ leaves the single private-run annotation that reconciliation reads."""
+ workflow = yaml.safe_load(read(".github/workflows/promote-private-pro-runtime.yml"))
+ step = workflow["jobs"]["promote"]["steps"][0]["run"]
+ script = step[step.index("wait_for_workflow() {"):]
+ source_step = "Require current exact main source before credentials"
+ fake_gh = r'''#!/usr/bin/env python3
+import json, os, subprocess, sys
+fix = os.environ["FIX"]
+args = sys.argv[1:]
+expression = args[args.index("--jq") + 1] if "--jq" in args else None
+with open(os.path.join(fix, "state.json")) as handle:
+ state = json.load(handle)
+
+def emit(value):
+ text = json.dumps(value)
+ if expression is not None:
+ text = subprocess.run(["jq", "-r", expression], input=text, text=True,
+ capture_output=True, check=True).stdout
+ sys.stdout.write(text if text.endswith("\n") else text + "\n")
+
+def attempt(run_id):
+ return state["attempts"][min(int(run_id) - 100, len(state["attempts"]) - 1)]
+
+if args[0] == "api" and "--method" in args:
+ sys.stdin.read()
+ run_id = 100 + len(state["dispatched"])
+ state["dispatched"].append(run_id)
+ with open(os.path.join(fix, "state.json"), "w") as handle:
+ json.dump(state, handle)
+ emit({"workflow_run_id": run_id})
+elif args[0] == "api" and args[1].endswith("/git/ref/heads/main"):
+ emit({"object": {"sha": state["main"]}})
+elif args[0] == "api" and "/actions/runs/" in args[1]:
+ emit({"head_sha": attempt(args[1].rsplit("/", 1)[1])["head"]})
+elif args[:2] == ["run", "view"] and "jobs" in args:
+ run = attempt(args[2])
+ jobs = [{"name": "credential_containment", "conclusion": "success",
+ "steps": [{"name": "Checkout pulse-pro", "conclusion": "success"}]}]
+ if run["failed_step"]:
+ jobs.append({"name": "promote", "conclusion": "failure", "steps": [
+ {"name": "Checkout pulse-pro", "conclusion": "success"},
+ {"name": run["failed_step"], "conclusion": "failure"}]})
+ emit({"jobs": jobs})
+elif args[:2] == ["run", "view"]:
+ emit({"status": "completed", "conclusion": attempt(args[2])["conclusion"],
+ "url": f"https://github.com/rcourtman/pulse-pro/actions/runs/{args[2]}"})
+else:
+ sys.exit(f"unexpected gh call: {args}")
+'''
+ old_main, new_main = "a" * 40, "b" * 40
+
+ def run(main: str, attempts: list[dict]) -> tuple[subprocess.CompletedProcess, list[int]]:
+ with tempfile.TemporaryDirectory() as directory:
+ fix = Path(directory)
+ (fix / "bin").mkdir()
+ for name, body in (("gh", fake_gh), ("sleep", "#!/bin/bash\n")):
+ (fix / "bin" / name).write_text(body)
+ (fix / "bin" / name).chmod(0o755)
+ (fix / "state.json").write_text(
+ json.dumps({"main": main, "attempts": attempts, "dispatched": []})
+ )
+ env = {**os.environ, "FIX": str(fix), "TAG": "v6.6.0-rc.6", "VERSION": "6.6.0-rc.6",
+ "IS_PRERELEASE": "true", "R2_PREFIX": "prefix", "PULSE_LEASE_SHA": "c" * 40,
+ "PULSE_CONVERGENCE_RUN_ID": "7", "PULSE_OWNER_ASSET_NAME": "owner.json",
+ "PULSE_OWNER_ASSET_SHA256": "d" * 64,
+ "PATH": f"{fix / 'bin'}{os.pathsep}{os.environ['PATH']}"}
+ result = subprocess.run(
+ ["bash", "--noprofile", "--norc", "-eu", "-o", "pipefail", "-c", script],
+ cwd=REPO_ROOT, env=env, text=True, capture_output=True, timeout=60,
+ )
+ dispatched = json.loads((fix / "state.json").read_text())["dispatched"]
+ return result, dispatched
+
+ refused = {"conclusion": "failure", "head": old_main, "failed_step": source_step}
+ promoted = {"conclusion": "success", "head": new_main, "failed_step": ""}
+
+ # A refusal because main moved after dispatch is re-dispatched and leaves no annotation.
+ result, dispatched = run(new_main, [refused, promoted])
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(dispatched, [100, 101])
+ self.assertNotIn("::error::", result.stdout)
+ self.assertIn("re-dispatching on the current main", result.stdout)
+
+ # The same refusal while main is unchanged is a source-read fault, not a race: terminal.
+ result, dispatched = run(old_main, [refused])
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(dispatched, [100])
+
+ # A failure past the source check may have used credentials: never re-dispatched.
+ mutated = {"conclusion": "failure", "head": old_main,
+ "failed_step": "Download signed packet and promote live broker"}
+ result, dispatched = run(new_main, [mutated])
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(dispatched, [100])
+ self.assertEqual(
+ re.findall(r"::error::.*", result.stdout),
+ ["::error::private Pro live promotion failed with conclusion=failure: "
+ "https://github.com/rcourtman/pulse-pro/actions/runs/100"],
+ )
+
+ # A cancelled run is not a refusal either.
+ result, dispatched = run(new_main, [{**refused, "conclusion": "cancelled"}])
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(dispatched, [100])
+
+ # Retries are bounded, and only the last attempt's URL is annotated.
+ result, dispatched = run(new_main, [refused])
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(dispatched, [100, 101, 102, 103, 104])
+ self.assertEqual(
+ re.findall(r"https://github\.com/rcourtman/pulse-pro/actions/runs/\d+",
+ "\n".join(re.findall(r"::error::.*", result.stdout))),
+ ["https://github.com/rcourtman/pulse-pro/actions/runs/104"],
+ )
+
def test_release_promotion_policy_requires_live_rc_and_v5_policy(self) -> None:
content = read("docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md")
self.assertIn("`beta.N` is the normal user-testing stage", content)