Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep reconciling preview when stable convergence evidence is gone
rcourtman43f0a21d1a25
diff --git a/scripts/release_control/reconcile_release_convergence.py b/scripts/release_control/reconcile_release_convergence.pyindex 6f40e4f832..eab3715b82 100644--- a/scripts/release_control/reconcile_release_convergence.py+++ b/scripts/release_control/reconcile_release_convergence.py@@ -617,7 +617,17 @@ def reconcile(github: GitHub, run_id: int, max_attempts: int) -> None:owner_run_id = validate_marker(marker, release=release, tag=tag, source_run_id=source_run_id)- owner_run = github.api(f"repos/{repository}/actions/runs/{owner_run_id}")+ try:+ owner_run = github.api(f"repos/{repository}/actions/runs/{owner_run_id}")+ except GitHubNotFound as exc:+ # The immutable marker outlives Actions history. Once its owner run is+ # deleted, release-convergence.yml can never adopt the tag (it waits+ # for that owner to read "completed"), so a retry would only time out.+ raise ReconciliationError(+ f"activation marker owner run {owner_run_id} for committed {tag} no longer "+ "exists in Actions history; convergence cannot adopt this release until an "+ "operator resolves it. No retry was dispatched."+ ) from excrequire_run_identity(owner_run,repository=repository,@@ -753,19 +763,29 @@ def main() -> int:)try:run_ids = discover(github) if args.latest else [args.run_id]- if not run_ids:- report_decision(- "No failed run selected for retry among current immutable channel heads. "- "Mutable releases and missing convergence runs are not qualified by this check."- )- return 0- for run_id in run_ids:- assert run_id is not None- reconcile(github, run_id, args.max_attempts)except (OSError, ReconciliationError) as exc:print(f"release convergence reconciliation failed: {exc}", file=sys.stderr)return 1- return 0+ if not run_ids:+ report_decision(+ "No failed run selected for retry among current immutable channel heads. "+ "Mutable releases and missing convergence runs are not qualified by this check."+ )+ return 0+ # Each selected run is a separate channel's debt. One channel's broken+ # evidence must not starve the other channel's retry; it still fails the job.+ failed = False+ for run_id in run_ids:+ assert run_id is not None+ try:+ reconcile(github, run_id, args.max_attempts)+ except (OSError, ReconciliationError) as exc:+ failed = True+ print(+ f"release convergence reconciliation failed for run {run_id}: {exc}",+ file=sys.stderr,+ )+ return 1 if failed else 0if __name__ == "__main__":diff --git a/scripts/release_control/reconcile_release_convergence_test.py b/scripts/release_control/reconcile_release_convergence_test.pyindex 6661aa02f3..081d3e2e5c 100644--- a/scripts/release_control/reconcile_release_convergence_test.py+++ b/scripts/release_control/reconcile_release_convergence_test.py@@ -744,6 +744,42 @@ class ReconciliationTests(unittest.TestCase):subject.reconcile(github, github.run_id, 5)self.assertEqual([], github.posts)+ def test_deleted_activation_owner_is_named_without_retry(self):+ github = FakeGitHub()+ fake_api = github.api++ def api(endpoint):+ if endpoint.endswith(f"/actions/runs/{github.owner_run_id}"):+ raise subject.GitHubNotFound("GitHub object was not found")+ return fake_api(endpoint)++ github.api = api+ with self.assertRaises(subject.ReconciliationError) as raised:+ subject.reconcile(github, github.run_id, 5)+ self.assertNotIsInstance(raised.exception, subject.GitHubNotFound)+ self.assertIn("owner run 90 for committed v6.5.0-rc.1", str(raised.exception))+ self.assertIn("no longer exists", str(raised.exception))+ self.assertEqual([], github.posts)++ def test_one_channel_failure_does_not_starve_the_other(self):+ args = argparse.Namespace(repository="rcourtman/Pulse", run_id=None,+ latest=True, max_attempts=5, dry_run=True)+ attempted = []++ def reconcile(github, run_id, max_attempts):+ attempted.append(run_id)+ if run_id == 1:+ raise subject.ReconciliationError("stable evidence is gone")++ stderr = io.StringIO()+ with patch.object(subject, "parse_args", return_value=args), \+ patch.object(subject, "discover", return_value=[1, 2]), \+ patch.object(subject, "reconcile", side_effect=reconcile), \+ contextlib.redirect_stderr(stderr):+ self.assertEqual(1, subject.main())+ self.assertEqual([1, 2], attempted)+ self.assertIn("failed for run 1: stable evidence is gone", stderr.getvalue())+def test_precommit_owner_is_renewed_without_using_committed_budget(self):github = FakeGitHub(committed=False)github.runs[0]["run_attempt"] = 6