Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep reconciling preview when stable convergence evidence is gone

rcourtman43f0a21d1a25
diff --git a/scripts/release_control/reconcile_release_convergence.py b/scripts/release_control/reconcile_release_convergence.py
index 6f40e4f832..eab3715b82 100644
--- a/scripts/release_control/reconcile_release_convergence.py
+++ b/scripts/release_control/reconcile_release_convergence.py
@@ -617,7 +617,17 @@ def reconcile(github: GitHub, run_id: int, max_attempts: int) -> None:
owner_run_id = validate_marker(
marker, release=release, tag=tag, source_run_id=source_run_id
)
- owner_run = github.api(f"repos/{repository}/actions/runs/{owner_run_id}")
+ try:
+ owner_run = github.api(f"repos/{repository}/actions/runs/{owner_run_id}")
+ except GitHubNotFound as exc:
+ # The immutable marker outlives Actions history. Once its owner run is
+ # deleted, release-convergence.yml can never adopt the tag (it waits
+ # for that owner to read "completed"), so a retry would only time out.
+ raise ReconciliationError(
+ f"activation marker owner run {owner_run_id} for committed {tag} no longer "
+ "exists in Actions history; convergence cannot adopt this release until an "
+ "operator resolves it. No retry was dispatched."
+ ) from exc
require_run_identity(
owner_run,
repository=repository,
@@ -753,19 +763,29 @@ def main() -> int:
)
try:
run_ids = discover(github) if args.latest else [args.run_id]
- if not run_ids:
- report_decision(
- "No failed run selected for retry among current immutable channel heads. "
- "Mutable releases and missing convergence runs are not qualified by this check."
- )
- return 0
- for run_id in run_ids:
- assert run_id is not None
- reconcile(github, run_id, args.max_attempts)
except (OSError, ReconciliationError) as exc:
print(f"release convergence reconciliation failed: {exc}", file=sys.stderr)
return 1
- return 0
+ if not run_ids:
+ report_decision(
+ "No failed run selected for retry among current immutable channel heads. "
+ "Mutable releases and missing convergence runs are not qualified by this check."
+ )
+ return 0
+ # Each selected run is a separate channel's debt. One channel's broken
+ # evidence must not starve the other channel's retry; it still fails the job.
+ failed = False
+ for run_id in run_ids:
+ assert run_id is not None
+ try:
+ reconcile(github, run_id, args.max_attempts)
+ except (OSError, ReconciliationError) as exc:
+ failed = True
+ print(
+ f"release convergence reconciliation failed for run {run_id}: {exc}",
+ file=sys.stderr,
+ )
+ return 1 if failed else 0
if __name__ == "__main__":
diff --git a/scripts/release_control/reconcile_release_convergence_test.py b/scripts/release_control/reconcile_release_convergence_test.py
index 6661aa02f3..081d3e2e5c 100644
--- a/scripts/release_control/reconcile_release_convergence_test.py
+++ b/scripts/release_control/reconcile_release_convergence_test.py
@@ -744,6 +744,42 @@ class ReconciliationTests(unittest.TestCase):
subject.reconcile(github, github.run_id, 5)
self.assertEqual([], github.posts)
+ def test_deleted_activation_owner_is_named_without_retry(self):
+ github = FakeGitHub()
+ fake_api = github.api
+
+ def api(endpoint):
+ if endpoint.endswith(f"/actions/runs/{github.owner_run_id}"):
+ raise subject.GitHubNotFound("GitHub object was not found")
+ return fake_api(endpoint)
+
+ github.api = api
+ with self.assertRaises(subject.ReconciliationError) as raised:
+ subject.reconcile(github, github.run_id, 5)
+ self.assertNotIsInstance(raised.exception, subject.GitHubNotFound)
+ self.assertIn("owner run 90 for committed v6.5.0-rc.1", str(raised.exception))
+ self.assertIn("no longer exists", str(raised.exception))
+ self.assertEqual([], github.posts)
+
+ def test_one_channel_failure_does_not_starve_the_other(self):
+ args = argparse.Namespace(repository="rcourtman/Pulse", run_id=None,
+ latest=True, max_attempts=5, dry_run=True)
+ attempted = []
+
+ def reconcile(github, run_id, max_attempts):
+ attempted.append(run_id)
+ if run_id == 1:
+ raise subject.ReconciliationError("stable evidence is gone")
+
+ stderr = io.StringIO()
+ with patch.object(subject, "parse_args", return_value=args), \
+ patch.object(subject, "discover", return_value=[1, 2]), \
+ patch.object(subject, "reconcile", side_effect=reconcile), \
+ contextlib.redirect_stderr(stderr):
+ self.assertEqual(1, subject.main())
+ self.assertEqual([1, 2], attempted)
+ self.assertIn("failed for run 1: stable evidence is gone", stderr.getvalue())
+
def test_precommit_owner_is_renewed_without_using_committed_budget(self):
github = FakeGitHub(committed=False)
github.runs[0]["run_attempt"] = 6