Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Recover lifecycle discovery from peer stream cancellation
pulse-triage[bot]aaf38867e700
diff --git a/.github/workflows/release-lifecycle-rehearsal.yml b/.github/workflows/release-lifecycle-rehearsal.ymlindex 801c89e029..3036b66cb4 100644--- a/.github/workflows/release-lifecycle-rehearsal.yml+++ b/.github/workflows/release-lifecycle-rehearsal.yml@@ -51,16 +51,10 @@ jobs:TO_INPUT: ${{ inputs.to_version }}run: |set -euo pipefail- releases_json="$(mktemp)"- gh api --paginate "repos/${REPOSITORY}/releases?per_page=100" \- | jq -s 'add // []' > "${releases_json}"- latest_tag="$(gh api "repos/${REPOSITORY}/releases/latest" --jq '.tag_name')"resolved="$(python3 scripts/release_lifecycle_rehearsal_versions.py \- --releases-json "${releases_json}" \- --latest-tag "${latest_tag}" \+ --repository "${REPOSITORY}" \--from "${FROM_INPUT}" \--to "${TO_INPUT}")"- rm -f "${releases_json}"from_tag="$(sed -n 's/^from_tag=//p' <<<"${resolved}")"to_tag="$(sed -n 's/^to_tag=//p' <<<"${resolved}")"reason="$(sed -n 's/^reason=//p' <<<"${resolved}")"diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 096e017bb4..85ba1a3d47 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -8145,6 +8145,17 @@ defaulted and nothing newer is published, the pair is the previous stable tothe latest stable. An explicit pair must be published and move forward, so asame-version or missing-target run fails instead of passing.+Live discovery uses that resolver's `--repository` mode. Each fixed metadata+GET has a 60-second deadline. Only exit 1 with the complete diagnostic+`stream error: stream ID <number>; CANCEL; received from peer` permits two+retries of the identical GET, after two and four seconds. Each failed+attempt's diagnostic remains visible and its partial output is discarded.+Paginated inventory must be complete array pages; malformed JSON/shape, access+refusals, mixed or unknown errors, timeouts and process cancellation stop+discovery without pair outputs or installation. An advertised latest-tag+override is unavailable in live mode. No installer, updater, rollback or+release action is retried by this policy.+`scripts/release_lifecycle_rehearsal.sh` defaults to running every phase inside the samedigest-pinned `jrei/systemd-debian:12` container as the install smoke.@@ -8199,7 +8210,11 @@ or altered seeded field fails `--check-snapshot`, and a deleted file or arewritten secret store fails `--compare-datadir`), the harness contract (the real updaterinvocation without bypass flags, the signed installer, the shared image digest,and per-phase identity, settings and data-dir checks) and the workflow trust-shape. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1`+shape. Discovery tests execute the actual workflow resolution shell and CLI+against a synthetic `gh`, proving recovered complete-page pair outputs and+no outputs on refusal or malformed data. Process controls cover exhaustion,+mixed errors, signals, deadlines and unchanged GET identity; these are not+installed-recovery or live-provider acceptance. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1`passed all four phases. An earlier run correctly failed phase 1 on a new failedunit (`systemd-hostnamed`), which led to the container-only list above. Thehosted Docker run is not yet claimed.diff --git a/scripts/release_lifecycle_rehearsal_versions.py b/scripts/release_lifecycle_rehearsal_versions.pyindex b7071992ee..d606c805a5 100755--- a/scripts/release_lifecycle_rehearsal_versions.py+++ b/scripts/release_lifecycle_rehearsal_versions.py@@ -20,7 +20,9 @@ import argparseimport functoolsimport jsonimport re+import subprocessimport sys+import timefrom dataclasses import dataclassfrom pathlib import Path@@ -42,6 +44,66 @@ class Selection:reason: str+def _github_release_json(repository: str, *, latest: bool = False) -> object:+ """Read only release metadata; never retry a refusal or an unknown error."""+ endpoint = f"repos/{repository}/releases"+ command = ["gh", "api", "--method", "GET"]+ if latest:+ command.append(f"{endpoint}/latest")+ else:+ command.extend(["--paginate", "--slurp", f"{endpoint}?per_page=100"])+ for attempt in range(1, 4):+ try:+ result = subprocess.run(+ command, capture_output=True, text=True, check=False, timeout=60,+ )+ except subprocess.TimeoutExpired as error:+ raise ResolutionError("GitHub release metadata read timed out; not retrying") from error+ if result.stderr:+ print(result.stderr, end="", file=sys.stderr)+ if result.returncode == 0:+ # A successful but malformed/incomplete response is not a+ # transport-only failure and must not select an older fallback.+ return json.loads(result.stdout)+ # Require the complete diagnostic to match the observed peer stream+ # cancellation. Any additional error (including 401/403/429), unknown+ # failure, timeout or signal is terminal. Discard partial stdout.+ if (+ result.returncode != 1+ or attempt == 3+ or re.fullmatch(+ r"stream error: stream ID [0-9]+; CANCEL; received from peer\n?",+ result.stderr,+ ) is None+ ):+ raise ResolutionError(+ f"GitHub release metadata read failed (exit {result.returncode}, attempt {attempt}/3)"+ )+ print(+ f"release lifecycle rehearsal: metadata stream cancelled by peer; "+ f"retrying the same GET ({attempt}/2)", file=sys.stderr,+ )+ time.sleep(attempt * 2)+ raise AssertionError("unreachable")+++def discover(repository: str) -> tuple[list[dict], str]:+ # No URL, host, endpoint, method, token or read-filter overrides. The+ # workflow supplies github.repository, not a dispatch input.+ if re.fullmatch(r"[A-Za-z0-9_-][A-Za-z0-9_.-]*/[A-Za-z0-9_-][A-Za-z0-9_.-]*", repository) is None:+ raise ResolutionError("invalid GitHub repository name")+ pages = _github_release_json(repository)+ if not isinstance(pages, list) or not pages or any(not isinstance(page, list) for page in pages):+ raise ResolutionError("GitHub release inventory must contain complete array pages")+ releases = [release for page in pages for release in page]+ if any(not isinstance(release, dict) for release in releases):+ raise ResolutionError("GitHub release inventory contains a non-object")+ latest = _github_release_json(repository, latest=True)+ if not isinstance(latest, dict) or not isinstance(latest.get("tag_name"), str):+ raise ResolutionError("GitHub latest release must contain a tag_name")+ return releases, latest["tag_name"]++def parse_tag(tag: str) -> tuple[tuple[int, int, int], tuple[str, ...]]:match = TAG_RE.fullmatch(tag)if not match:@@ -152,21 +214,29 @@ def resolve(def main(argv: list[str] | None = None) -> int:parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])- parser.add_argument("--releases-json", required=True, type=Path,- help="JSON array of GitHub release objects (all pages)")+ source = parser.add_mutually_exclusive_group(required=True)+ source.add_argument("--releases-json", type=Path,+ help="offline JSON array of GitHub release objects (all pages)")+ source.add_argument("--repository", help="read published release metadata from this GitHub repository")parser.add_argument("--latest-tag", default="",- help="tag_name of GET /repos/{repo}/releases/latest")+ help="tag_name of GET /repos/{repo}/releases/latest (offline mode only)")parser.add_argument("--from", dest="from_version", default="")parser.add_argument("--to", dest="to_version", default="")args = parser.parse_args(argv)+ if args.repository and args.latest_tag:+ parser.error("--latest-tag cannot override live --repository discovery")try:- releases = json.loads(args.releases_json.read_text(encoding="utf-8"))+ if args.repository:+ releases, latest_tag = discover(args.repository)+ else:+ releases = json.loads(args.releases_json.read_text(encoding="utf-8"))+ latest_tag = args.latest_tag.strip()if not isinstance(releases, list):raise ResolutionError("releases JSON must be an array")selection = resolve(releases,- args.latest_tag.strip(),+ latest_tag,args.from_version.strip(),args.to_version.strip(),)diff --git a/scripts/tests/test_release_lifecycle_rehearsal.py b/scripts/tests/test_release_lifecycle_rehearsal.pyindex 00131f2f5b..5aa7fdf014 100644--- a/scripts/tests/test_release_lifecycle_rehearsal.py+++ b/scripts/tests/test_release_lifecycle_rehearsal.py@@ -4,14 +4,18 @@from __future__ import annotationsimport importlib.util+import contextlib+import ioimport jsonimport osimport reimport subprocessimport sysimport tempfile+import textwrapimport unittestfrom pathlib import Path+from unittest.mock import patchREPO_ROOT = Path(__file__).resolve().parents[2]@@ -118,6 +122,176 @@ class ResolvePairTest(unittest.TestCase):self.assertIn("to_tag=v6.4.1\n", result.stdout)+class DiscoveryTest(unittest.TestCase):+ stream_error = "stream error: stream ID 1; CANCEL; received from peer\n"+ inventory_command = ["gh", "api", "--method", "GET", "--paginate", "--slurp",+ "repos/rcourtman/Pulse/releases?per_page=100"]+ latest_command = ["gh", "api", "--method", "GET", "repos/rcourtman/Pulse/releases/latest"]+ pages = [[release("v6.5.0"), release("v6.6.0")], [release("v6.7.0-beta.1")]]++ @staticmethod+ def result(status=0, stdout="", stderr=""):+ return subprocess.CompletedProcess([], status, stdout=stdout, stderr=stderr)++ def test_recovered_inventory_uses_only_complete_fresh_pages(self):+ results = [+ self.result(1, json.dumps([[release("v99.0.0")]]), self.stream_error),+ self.result(stdout=json.dumps(self.pages)),+ self.result(stdout=json.dumps(release("v6.6.0"))),+ ]+ errors = io.StringIO()+ with patch.object(versions.subprocess, "run", side_effect=results) as command, \+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(errors):+ releases, latest = versions.discover("rcourtman/Pulse")+ self.assertEqual([self.inventory_command, self.inventory_command, self.latest_command],+ [call.args[0] for call in command.call_args_list])+ self.assertTrue(all(call.kwargs["timeout"] == 60 for call in command.call_args_list))+ sleep.assert_called_once_with(2)+ self.assertIn(self.stream_error, errors.getvalue())+ self.assertEqual(versions.resolve(releases, latest).to_tag, "v6.7.0-beta.1")++ def test_latest_read_retries_without_reloading_inventory(self):+ results = [self.result(stdout=json.dumps(self.pages)),+ self.result(1, "partial latest", self.stream_error),+ self.result(stdout=json.dumps(release("v6.6.0")))]+ with patch.object(versions.subprocess, "run", side_effect=results) as command, \+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(io.StringIO()):+ self.assertEqual(versions.discover("rcourtman/Pulse")[1], "v6.6.0")+ self.assertEqual([self.inventory_command, self.latest_command, self.latest_command],+ [call.args[0] for call in command.call_args_list])+ sleep.assert_called_once_with(2)++ def test_retry_budget_exhaustion_emits_no_pair(self):+ output, errors = io.StringIO(), io.StringIO()+ with patch.object(versions.subprocess, "run", return_value=self.result(1, "[]", self.stream_error)) as command, \+ patch.object(versions.time, "sleep") as sleep, \+ contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors):+ self.assertEqual(versions.main(["--repository", "rcourtman/Pulse"]), 1)+ self.assertEqual(command.call_count, 3)+ self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 4])+ self.assertEqual(errors.getvalue().count(self.stream_error), 3)+ self.assertIn("attempt 3/3", errors.getvalue())+ self.assertEqual(output.getvalue(), "")++ def test_access_unknown_and_mixed_errors_never_retry(self):+ for detail in ("HTTP 401", "HTTP 403", "HTTP 404", "HTTP 429", "HTTP 503",+ "capability refused", "connection refused", "", "unavailable",+ self.stream_error.replace("CANCEL", "REFUSED_STREAM")):+ for error in (detail, self.stream_error + detail + "\n"):+ with self.subTest(error=error), \+ patch.object(versions.subprocess, "run", return_value=self.result(1, "[]", error)) as command, \+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(io.StringIO()):+ with self.assertRaises(versions.ResolutionError):+ versions.discover("rcourtman/Pulse")+ self.assertEqual(command.call_count, 1)+ sleep.assert_not_called()++ def test_signal_cancellation_and_other_exits_never_retry(self):+ for status in (-15, -2, 2, 124, 130, 137, 143):+ with self.subTest(status=status), \+ patch.object(versions.subprocess, "run", return_value=self.result(status, "[]", self.stream_error)) as command, \+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(io.StringIO()):+ with self.assertRaisesRegex(versions.ResolutionError, f"exit {status}"):+ versions.discover("rcourtman/Pulse")+ self.assertEqual(command.call_count, 1)+ sleep.assert_not_called()++ def test_timeout_never_retries(self):+ with patch.object(versions.subprocess, "run", side_effect=subprocess.TimeoutExpired("gh", 60)) as command, \+ patch.object(versions.time, "sleep") as sleep:+ with self.assertRaisesRegex(versions.ResolutionError, "timed out; not retrying"):+ versions.discover("rcourtman/Pulse")+ self.assertEqual(command.call_count, 1)+ sleep.assert_not_called()++ def test_malformed_inventory_stops_before_latest_read(self):+ for payload in ("not json", "{}", "[]", '[{"tag_name":"v6.6.0"}]', '[[],{}]', '[[null]]'):+ with self.subTest(payload=payload), \+ patch.object(versions.subprocess, "run", return_value=self.result(stdout=payload)) as command, \+ patch.object(versions.time, "sleep") as sleep:+ with self.assertRaises((versions.ResolutionError, json.JSONDecodeError)):+ versions.discover("rcourtman/Pulse")+ self.assertEqual(command.call_count, 1)+ sleep.assert_not_called()++ def test_malformed_latest_stops_without_retry(self):+ for payload in ("not json", "[]", "{}", '{"tag_name":null}'):+ with self.subTest(payload=payload), \+ patch.object(versions.subprocess, "run", side_effect=[+ self.result(stdout=json.dumps(self.pages)), self.result(stdout=payload)]) as command, \+ patch.object(versions.time, "sleep") as sleep:+ with self.assertRaises((versions.ResolutionError, json.JSONDecodeError)):+ versions.discover("rcourtman/Pulse")+ self.assertEqual(command.call_count, 2)+ sleep.assert_not_called()++ def test_invalid_repository_and_live_latest_override_do_not_start_reads(self):+ with patch.object(versions.subprocess, "run") as command, \+ contextlib.redirect_stderr(io.StringIO()):+ for repository in ("../Pulse", "rcourtman/Pulse/other", "https://example.com", "-H", "rcourtman/Pulse\n"):+ with self.subTest(repository=repository), self.assertRaises(versions.ResolutionError):+ versions.discover(repository)+ with self.assertRaises(SystemExit) as error:+ versions.main(["--repository", "rcourtman/Pulse", "--latest-tag", "v6.5.0"])+ self.assertEqual(error.exception.code, 2)+ command.assert_not_called()++ def workflow_resolution(self, outcomes, workflow=None):+ # Execute the actual workflow shell and Python CLI, replacing only gh+ # with a process fixture. No installation, network or token is used.+ if workflow is None:+ workflow = WORKFLOW_PATH.read_text(encoding="utf-8")+ step = workflow.split(" - name: Resolve published release pair\n", 1)[1].split("\n - name:", 1)[0]+ body = textwrap.dedent(step.split(" run: |\n", 1)[1])+ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ (root / "outcomes.json").write_text(json.dumps(outcomes))+ gh = root / "gh"+ gh.write_text(f"#!{sys.executable}\n" + '''import json, os, pathlib, sys+root = pathlib.Path(os.environ['DISCOVERY_FIXTURE'])+with (root / 'calls.jsonl').open('a') as out:+ out.write(json.dumps(sys.argv[1:]) + '\\n')+count = len((root / 'calls.jsonl').read_text().splitlines())+outcomes = json.loads((root / 'outcomes.json').read_text())+status, output, error = outcomes[min(count - 1, len(outcomes) - 1)]+print(output, end='', file=sys.stdout)+print(error, end='', file=sys.stderr)+sys.exit(status)+''')+ gh.chmod(0o755)+ output, summary = root / "outputs", root / "summary"+ result = subprocess.run(["bash", "-c", body], cwd=REPO_ROOT, capture_output=True,+ text=True, timeout=15, env={**os.environ,+ "PATH": str(root) + os.pathsep + os.environ["PATH"],+ "DISCOVERY_FIXTURE": str(root), "GH_TOKEN": "synthetic",+ "REPOSITORY": "rcourtman/Pulse", "FROM_INPUT": "", "TO_INPUT": "",+ "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(summary)})+ calls = [json.loads(line) for line in (root / "calls.jsonl").read_text().splitlines()]+ return (result, calls, output.read_text() if output.exists() else "",+ summary.read_text() if summary.exists() else "")++ def test_workflow_recovers_only_metadata_and_writes_real_pair_outputs(self):+ result, calls, output, summary = self.workflow_resolution([+ (1, json.dumps([[release("v99.0.0")]]), self.stream_error),+ (0, json.dumps(self.pages), ""), (0, json.dumps(release("v6.6.0")), ""),+ ])+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual([self.inventory_command[1:], self.inventory_command[1:], self.latest_command[1:]], calls)+ self.assertRegex(output, r"from_tag<<[^\n]+\nv6\.6\.0\n")+ self.assertRegex(output, r"to_tag<<[^\n]+\nv6\.7\.0-beta\.1\n")+ self.assertNotIn("v99.0.0", output + summary)+ self.assertIn("v6.7.0-beta.1", summary)+ self.assertIn(self.stream_error, result.stderr)++ def test_workflow_refusal_and_malformed_metadata_emit_no_outputs(self):+ for outcome in ((1, "[]", "gh: Forbidden (HTTP 403)\n"), (0, "not json", "")):+ with self.subTest(outcome=outcome):+ result, calls, output, summary = self.workflow_resolution([outcome])+ self.assertEqual(result.returncode, 1, result.stderr)+ self.assertEqual(calls, [self.inventory_command[1:]])+ self.assertEqual(output + summary, "")++class HarnessContractTest(unittest.TestCase):harness = HARNESS_PATH.read_text(encoding="utf-8")