Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Treat a deleted convergence owner as terminal
rcourtmanca3812e5ed02
diff --git a/.github/workflows/release-convergence.yml b/.github/workflows/release-convergence.ymlindex 736580dfee..fd4fa02505 100644--- a/.github/workflows/release-convergence.yml+++ b/.github/workflows/release-convergence.yml@@ -130,18 +130,35 @@ jobs:fiactivation_owner_run_id="$(jq -r '.convergence_run_id' "${marker}")"if [ "${activation_owner_run_id}" != "${EXPECTED_CONVERGENCE_RUN_ID}" ]; then- original_status="$(+ # The immutable marker outlives Actions history: runs are+ # deleted by retention or by hand, and GitHub only deletes a+ # run once it has completed or is older than two weeks, far+ # beyond every job timeout here. A definite 404 is therefore+ # a terminal owner. Any other read failure is unknown and+ # keeps waiting.+ owner_read_error="$(mktemp)"+ if ! original_status="$(gh api \"repos/${{ github.repository }}/actions/runs/${activation_owner_run_id}" \- --jq '.status' 2>/dev/null || true- )"- if [ "${original_status}" != "completed" ]; then+ --jq '.status' 2>"${owner_read_error}"+ )"; then+ original_status=""+ if grep -q 'HTTP 404' "${owner_read_error}"; then+ original_status="deleted"+ fi+ fi+ rm -f "${owner_read_error}"+ if [ "${original_status}" != "completed" ] && [ "${original_status}" != "deleted" ]; thenrm -f "${marker}"echo "Original convergence owner ${activation_owner_run_id} is ${original_status:-unavailable}; successor adoption is not yet allowed."sleep 2continuefi- echo "[OK] Adopting committed ${TAG} from completed convergence owner ${activation_owner_run_id}."+ if [ "${original_status}" = "deleted" ]; then+ echo "[OK] Adopting committed ${TAG}; convergence owner ${activation_owner_run_id} was deleted from Actions history and is terminal."+ else+ echo "[OK] Adopting committed ${TAG} from completed convergence owner ${activation_owner_run_id}."+ fifiactivation_marker_sha256="$(sha256sum "${marker}" | awk '{print $1}')"server_image_digest="$(jq -r '.server_image_digest' "${marker}")"diff --git a/.github/workflows/retry-release-convergence.yml b/.github/workflows/retry-release-convergence.ymlindex ca1d8afc14..23785326e3 100644--- a/.github/workflows/retry-release-convergence.yml+++ b/.github/workflows/retry-release-convergence.yml@@ -6,17 +6,17 @@ run-name: Reconcile release convergence debt# when the failed run predates a repair, so the complete convergence workflow# reacquires the global lease and replays every idempotent surface. Successful# surfaces are safe to repeat; retrying only failed jobs would bypass the lease.+#+# The hourly schedule is the retry mechanism. There is deliberately no+# workflow_run trigger: create-release.yml and this reconciler dispatch+# convergence with GITHUB_TOKEN, and GitHub does not deliver workflow_run for+# runs started that way. A workflow_run job only ever fired for convergence+# runs an operator started by hand; between 29 Aug and 24 Oct 2026 it never+# fired at all, while dozens of failed bot-dispatched runs were recovered by+# this schedule.on:- workflow_run:- workflows:- - Release Convergence- types:- - completed- branches:- - mainschedule:- # workflow_run delivery is not durable. Reconcile the current stable and- # preview channel heads hourly so a dropped event cannot strand aliases.+ # Reconcile the current stable and preview channel heads hourly.- cron: '37 * * * *'workflow_dispatch:@@ -29,28 +29,7 @@ concurrency:cancel-in-progress: falsejobs:- retry_committed_convergence:- if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' }}- runs-on: ubuntu-24.04- timeout-minutes: 10- steps:- - name: Checkout current reconciliation controls- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- with:- persist-credentials: false-- - name: Reconcile the failed convergence- env:- GH_TOKEN: ${{ github.token }}- PRO_REPOSITORY_TOKEN: ${{ secrets.WORKFLOW_PAT }}- RUN_ID: ${{ github.event.workflow_run.id }}- run: >-- python3 scripts/release_control/reconcile_release_convergence.py- --repository "${GITHUB_REPOSITORY}"- --run-id "${RUN_ID}"-- reconcile_missed_event:- if: ${{ github.event_name != 'workflow_run' }}+ reconcile_channel_heads:runs-on: ubuntu-24.04timeout-minutes: 10steps:diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 34d56af2f4..caaddd190e 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -3466,13 +3466,19 @@ artifact-selection behaviour.Reading a floating ref or a clobbered constant record is forbidden becausecached prior bytes could authorize stale ownership.Convergence retry is a current-control reconciliation, not unconditional- replay of the failed run's checkout. A terminal failure delivered through- `workflow_run` must be reconciled, and an hourly or manual pass must inspect- the current immutable stable and preview channel heads so a missed event- cannot strand mutable aliases. Discovery must not fall back behind a mutable+ replay of the failed run's checkout. An hourly or manual pass must inspect+ the current immutable stable and preview channel heads; that schedule is the+ retry mechanism, because GitHub does not deliver `workflow_run` for+ convergence runs dispatched with `GITHUB_TOKEN`, and no event-driven retry+ path may be presented as one. Discovery must not fall back behind a mutablechannel head. Before mutation, reconciliation must bind the exact mainlineworkflow-dispatch identity, release, source run, activation marker and- original convergence owner; reject malformed marker digests; suppress a+ original convergence owner. The immutable marker outlives Actions history:+ an original owner that GitHub definitively reports as not found is terminal,+ because GitHub deletes only completed runs or runs older than two weeks, so+ both the reconciler and the successor's adoption wait proceed from the+ marker alone, while any other owner read failure keeps waiting or fails.+ Reconciliation must also reject malformed marker digests; suppress anewer matching run; and enforce one aggregate attempt budget across rerunsand successor runs. If the failed run used the current default-branch commit,the complete run is rerun. If its checkout predates current controls, a freshdiff --git a/scripts/release_control/reconcile_release_convergence.py b/scripts/release_control/reconcile_release_convergence.pyindex e85168963d..44dd7f2caf 100644--- a/scripts/release_control/reconcile_release_convergence.py+++ b/scripts/release_control/reconcile_release_convergence.py@@ -637,24 +637,26 @@ def reconcile(github: GitHub, run_id: int, max_attempts: int) -> None:)try:owner_run = github.api(f"repos/{repository}/actions/runs/{owner_run_id}")- except GitHubNotFound as exc:- # The immutable marker outlives Actions history. Once its owner run is- # deleted, release-convergence.yml can never adopt the tag (it waits- # for that owner to read "completed"), so a retry would only time out.- raise ReconciliationError(- f"activation marker owner run {owner_run_id} for committed {tag} no longer "- "exists in Actions history; convergence cannot adopt this release until an "- "operator resolves it. No retry was dispatched."- ) from exc- require_run_identity(- owner_run,- repository=repository,- workflow_id=convergence_workflow_id,- path=CONVERGENCE_PATH,- display_title=title,- )- if owner_run.get("status") != "completed":- raise ReconciliationError("activation marker owner is not terminal")+ except GitHubNotFound:+ # The immutable marker outlives Actions history. GitHub only deletes a+ # run once it has completed or is older than two weeks, beyond every+ # convergence job timeout, so a deleted owner is terminal and+ # release-convergence.yml adopts the tag from it. The marker itself+ # was validated above; there is no run left to bind its identity to.+ print(+ f"Activation marker owner run {owner_run_id} for committed {tag} was "+ "deleted from Actions history; treating it as terminal."+ )+ else:+ require_run_identity(+ owner_run,+ repository=repository,+ workflow_id=convergence_workflow_id,+ path=CONVERGENCE_PATH,+ display_title=title,+ )+ if owner_run.get("status") != "completed":+ raise ReconciliationError("activation marker owner is not terminal")default_commit = github.api(f"repos/{repository}/commits/{default_branch}").get("sha")if not isinstance(default_commit, str) or EXACT_SHA.fullmatch(default_commit) is None:diff --git a/scripts/release_control/reconcile_release_convergence_test.py b/scripts/release_control/reconcile_release_convergence_test.pyindex f75bb5e061..c68e6e15d0 100644--- a/scripts/release_control/reconcile_release_convergence_test.py+++ b/scripts/release_control/reconcile_release_convergence_test.py@@ -798,21 +798,45 @@ class ReconciliationTests(unittest.TestCase):subject.reconcile(github, github.run_id, 5)self.assertEqual([], github.posts)- def test_deleted_activation_owner_is_named_without_retry(self):- github = FakeGitHub()+ @staticmethod+ def delete_owner_run(github, error):fake_api = github.apidef api(endpoint):if endpoint.endswith(f"/actions/runs/{github.owner_run_id}"):- raise subject.GitHubNotFound("GitHub object was not found")+ raise errorreturn fake_api(endpoint)github.api = api- with self.assertRaises(subject.ReconciliationError) as raised:++ def test_deleted_activation_owner_is_terminal_for_fresh_controls(self):+ # v6.5.0's owner run was deleted from Actions on 13 Oct 2026; the+ # immutable marker still names it. Deletion implies a terminal run.+ github = FakeGitHub()+ self.delete_owner_run(github, subject.GitHubNotFound("GitHub object was not found"))+ with contextlib.redirect_stdout(io.StringIO()) as output:+ subject.reconcile(github, github.run_id, 5)+ self.assertIn("owner run 90 for committed v6.5.0-rc.1 was deleted", output.getvalue())+ self.assertEqual(1, len(github.posts))+ endpoint, payload = github.posts[0]+ self.assertTrue(endpoint.endswith("/release-convergence.yml/dispatches"))+ self.assertEqual("200", payload["inputs"]["source_release_run_id"])++ def test_deleted_activation_owner_reruns_current_controls(self):+ github = FakeGitHub(current_controls=True)+ self.delete_owner_run(github, subject.GitHubNotFound("GitHub object was not found"))+ with contextlib.redirect_stdout(io.StringIO()):+ subject.reconcile(github, github.run_id, 5)+ self.assertEqual(+ [(f"repos/{github.repository}/actions/runs/{github.run_id}/rerun", None)],+ github.posts,+ )++ def test_unreadable_activation_owner_is_not_treated_as_deleted(self):+ github = FakeGitHub()+ self.delete_owner_run(github, subject.ReconciliationError("GitHub command failed"))+ with self.assertRaises(subject.ReconciliationError):subject.reconcile(github, github.run_id, 5)- self.assertNotIsInstance(raised.exception, subject.GitHubNotFound)- self.assertIn("owner run 90 for committed v6.5.0-rc.1", str(raised.exception))- self.assertIn("no longer exists", str(raised.exception))self.assertEqual([], github.posts)def test_one_channel_failure_does_not_starve_the_other(self):diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.pyindex 53a53a3842..25932f0489 100644--- a/scripts/release_control/release_promotion_policy_test.py+++ b/scripts/release_control/release_promotion_policy_test.py@@ -904,13 +904,14 @@ class ReleasePromotionPolicyTest(unittest.TestCase):self.assertIn(surface_name, convergence_verdict)self.assertIn("the committed release remains public", convergence_verdict)- self.assertIn("github.event.workflow_run.conclusion != 'success'", retry)+ # GitHub does not deliver workflow_run for GITHUB_TOKEN-dispatched+ # convergence runs, so the hourly schedule is the retry mechanism.+ self.assertNotIn("workflow_run", yaml.safe_load(retry)[True])self.assertIn("cron: '37 * * * *'", retry)self.assertIn("workflow_dispatch:", retry)self.assertIn("reconcile_release_convergence.py", retry)- self.assertIn('--run-id "${RUN_ID}"', retry)self.assertIn("--latest", retry)- self.assertEqual(2, retry.count("PRO_REPOSITORY_TOKEN: ${{ secrets.WORKFLOW_PAT }}"))+ self.assertEqual(1, retry.count("PRO_REPOSITORY_TOKEN: ${{ secrets.WORKFLOW_PAT }}"))self.assertNotIn("rerun-failed-jobs", retry)self.assertIn('actions/runs/{run_id}/rerun', reconciler)self.assertIn('release-convergence.yml/dispatches', reconciler)@@ -1141,10 +1142,14 @@ class ReleasePromotionPolicyTest(unittest.TestCase):self.assertIn("does not match the expected immutable release identity", await_commit)self.assertIn("EXPECTED_CONVERGENCE_RUN_ID: ${{ github.run_id }}", await_commit)self.assertIn('original_status}" != "completed"', await_commit)+ # A deleted owner is terminal: GitHub only deletes completed runs or+ # runs older than two weeks. Other read failures keep waiting.+ self.assertIn("grep -q 'HTTP 404'", await_commit)+ self.assertIn('original_status="deleted"', await_commit)self.assertIn("successor adoption is not yet allowed", await_commit)self.assertIn("activation_marker_sha256", await_commit)- self.assertIn("--run-id", retry)+ self.assertIn("--latest", retry)self.assertIn("DISPLAY_TITLE = re.compile", reconciler)self.assertIn("source_run_id = int(source_text)", reconciler)self.assertIn('actions/runs/{source_run_id}', reconciler)