Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Recover lifecycle discovery from peer stream cancellation

pulse-triage[bot]aaf38867e700
diff --git a/.github/workflows/release-lifecycle-rehearsal.yml b/.github/workflows/release-lifecycle-rehearsal.yml
index 801c89e029..3036b66cb4 100644
--- a/.github/workflows/release-lifecycle-rehearsal.yml
+++ b/.github/workflows/release-lifecycle-rehearsal.yml
@@ -51,16 +51,10 @@ jobs:
TO_INPUT: ${{ inputs.to_version }}
run: |
set -euo pipefail
- releases_json="$(mktemp)"
- gh api --paginate "repos/${REPOSITORY}/releases?per_page=100" \
- | jq -s 'add // []' > "${releases_json}"
- latest_tag="$(gh api "repos/${REPOSITORY}/releases/latest" --jq '.tag_name')"
resolved="$(python3 scripts/release_lifecycle_rehearsal_versions.py \
- --releases-json "${releases_json}" \
- --latest-tag "${latest_tag}" \
+ --repository "${REPOSITORY}" \
--from "${FROM_INPUT}" \
--to "${TO_INPUT}")"
- rm -f "${releases_json}"
from_tag="$(sed -n 's/^from_tag=//p' <<<"${resolved}")"
to_tag="$(sed -n 's/^to_tag=//p' <<<"${resolved}")"
reason="$(sed -n 's/^reason=//p' <<<"${resolved}")"
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 096e017bb4..85ba1a3d47 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -8145,6 +8145,17 @@ defaulted and nothing newer is published, the pair is the previous stable to
the latest stable. An explicit pair must be published and move forward, so a
same-version or missing-target run fails instead of passing.
+Live discovery uses that resolver's `--repository` mode. Each fixed metadata
+GET has a 60-second deadline. Only exit 1 with the complete diagnostic
+`stream error: stream ID <number>; CANCEL; received from peer` permits two
+retries of the identical GET, after two and four seconds. Each failed
+attempt's diagnostic remains visible and its partial output is discarded.
+Paginated inventory must be complete array pages; malformed JSON/shape, access
+refusals, mixed or unknown errors, timeouts and process cancellation stop
+discovery without pair outputs or installation. An advertised latest-tag
+override is unavailable in live mode. No installer, updater, rollback or
+release action is retried by this policy.
+
`scripts/release_lifecycle_rehearsal.sh` defaults to running every phase inside the same
digest-pinned `jrei/systemd-debian:12` container as the install smoke.
@@ -8199,7 +8210,11 @@ or altered seeded field fails `--check-snapshot`, and a deleted file or a
rewritten secret store fails `--compare-datadir`), the harness contract (the real updater
invocation without bypass flags, the signed installer, the shared image digest,
and per-phase identity, settings and data-dir checks) and the workflow trust
-shape. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1`
+shape. Discovery tests execute the actual workflow resolution shell and CLI
+against a synthetic `gh`, proving recovered complete-page pair outputs and
+no outputs on refusal or malformed data. Process controls cover exhaustion,
+mixed errors, signals, deadlines and unchanged GET identity; these are not
+installed-recovery or live-provider acceptance. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1`
passed all four phases. An earlier run correctly failed phase 1 on a new failed
unit (`systemd-hostnamed`), which led to the container-only list above. The
hosted Docker run is not yet claimed.
diff --git a/scripts/release_lifecycle_rehearsal_versions.py b/scripts/release_lifecycle_rehearsal_versions.py
index b7071992ee..d606c805a5 100755
--- a/scripts/release_lifecycle_rehearsal_versions.py
+++ b/scripts/release_lifecycle_rehearsal_versions.py
@@ -20,7 +20,9 @@ import argparse
import functools
import json
import re
+import subprocess
import sys
+import time
from dataclasses import dataclass
from pathlib import Path
@@ -42,6 +44,66 @@ class Selection:
reason: str
+def _github_release_json(repository: str, *, latest: bool = False) -> object:
+ """Read only release metadata; never retry a refusal or an unknown error."""
+ endpoint = f"repos/{repository}/releases"
+ command = ["gh", "api", "--method", "GET"]
+ if latest:
+ command.append(f"{endpoint}/latest")
+ else:
+ command.extend(["--paginate", "--slurp", f"{endpoint}?per_page=100"])
+ for attempt in range(1, 4):
+ try:
+ result = subprocess.run(
+ command, capture_output=True, text=True, check=False, timeout=60,
+ )
+ except subprocess.TimeoutExpired as error:
+ raise ResolutionError("GitHub release metadata read timed out; not retrying") from error
+ if result.stderr:
+ print(result.stderr, end="", file=sys.stderr)
+ if result.returncode == 0:
+ # A successful but malformed/incomplete response is not a
+ # transport-only failure and must not select an older fallback.
+ return json.loads(result.stdout)
+ # Require the complete diagnostic to match the observed peer stream
+ # cancellation. Any additional error (including 401/403/429), unknown
+ # failure, timeout or signal is terminal. Discard partial stdout.
+ if (
+ result.returncode != 1
+ or attempt == 3
+ or re.fullmatch(
+ r"stream error: stream ID [0-9]+; CANCEL; received from peer\n?",
+ result.stderr,
+ ) is None
+ ):
+ raise ResolutionError(
+ f"GitHub release metadata read failed (exit {result.returncode}, attempt {attempt}/3)"
+ )
+ print(
+ f"release lifecycle rehearsal: metadata stream cancelled by peer; "
+ f"retrying the same GET ({attempt}/2)", file=sys.stderr,
+ )
+ time.sleep(attempt * 2)
+ raise AssertionError("unreachable")
+
+
+def discover(repository: str) -> tuple[list[dict], str]:
+ # No URL, host, endpoint, method, token or read-filter overrides. The
+ # workflow supplies github.repository, not a dispatch input.
+ if re.fullmatch(r"[A-Za-z0-9_-][A-Za-z0-9_.-]*/[A-Za-z0-9_-][A-Za-z0-9_.-]*", repository) is None:
+ raise ResolutionError("invalid GitHub repository name")
+ pages = _github_release_json(repository)
+ if not isinstance(pages, list) or not pages or any(not isinstance(page, list) for page in pages):
+ raise ResolutionError("GitHub release inventory must contain complete array pages")
+ releases = [release for page in pages for release in page]
+ if any(not isinstance(release, dict) for release in releases):
+ raise ResolutionError("GitHub release inventory contains a non-object")
+ latest = _github_release_json(repository, latest=True)
+ if not isinstance(latest, dict) or not isinstance(latest.get("tag_name"), str):
+ raise ResolutionError("GitHub latest release must contain a tag_name")
+ return releases, latest["tag_name"]
+
+
def parse_tag(tag: str) -> tuple[tuple[int, int, int], tuple[str, ...]]:
match = TAG_RE.fullmatch(tag)
if not match:
@@ -152,21 +214,29 @@ def resolve(
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
- parser.add_argument("--releases-json", required=True, type=Path,
- help="JSON array of GitHub release objects (all pages)")
+ source = parser.add_mutually_exclusive_group(required=True)
+ source.add_argument("--releases-json", type=Path,
+ help="offline JSON array of GitHub release objects (all pages)")
+ source.add_argument("--repository", help="read published release metadata from this GitHub repository")
parser.add_argument("--latest-tag", default="",
- help="tag_name of GET /repos/{repo}/releases/latest")
+ help="tag_name of GET /repos/{repo}/releases/latest (offline mode only)")
parser.add_argument("--from", dest="from_version", default="")
parser.add_argument("--to", dest="to_version", default="")
args = parser.parse_args(argv)
+ if args.repository and args.latest_tag:
+ parser.error("--latest-tag cannot override live --repository discovery")
try:
- releases = json.loads(args.releases_json.read_text(encoding="utf-8"))
+ if args.repository:
+ releases, latest_tag = discover(args.repository)
+ else:
+ releases = json.loads(args.releases_json.read_text(encoding="utf-8"))
+ latest_tag = args.latest_tag.strip()
if not isinstance(releases, list):
raise ResolutionError("releases JSON must be an array")
selection = resolve(
releases,
- args.latest_tag.strip(),
+ latest_tag,
args.from_version.strip(),
args.to_version.strip(),
)
diff --git a/scripts/tests/test_release_lifecycle_rehearsal.py b/scripts/tests/test_release_lifecycle_rehearsal.py
index 00131f2f5b..5aa7fdf014 100644
--- a/scripts/tests/test_release_lifecycle_rehearsal.py
+++ b/scripts/tests/test_release_lifecycle_rehearsal.py
@@ -4,14 +4,18 @@
from __future__ import annotations
import importlib.util
+import contextlib
+import io
import json
import os
import re
import subprocess
import sys
import tempfile
+import textwrap
import unittest
from pathlib import Path
+from unittest.mock import patch
REPO_ROOT = Path(__file__).resolve().parents[2]
@@ -118,6 +122,176 @@ class ResolvePairTest(unittest.TestCase):
self.assertIn("to_tag=v6.4.1\n", result.stdout)
+class DiscoveryTest(unittest.TestCase):
+ stream_error = "stream error: stream ID 1; CANCEL; received from peer\n"
+ inventory_command = ["gh", "api", "--method", "GET", "--paginate", "--slurp",
+ "repos/rcourtman/Pulse/releases?per_page=100"]
+ latest_command = ["gh", "api", "--method", "GET", "repos/rcourtman/Pulse/releases/latest"]
+ pages = [[release("v6.5.0"), release("v6.6.0")], [release("v6.7.0-beta.1")]]
+
+ @staticmethod
+ def result(status=0, stdout="", stderr=""):
+ return subprocess.CompletedProcess([], status, stdout=stdout, stderr=stderr)
+
+ def test_recovered_inventory_uses_only_complete_fresh_pages(self):
+ results = [
+ self.result(1, json.dumps([[release("v99.0.0")]]), self.stream_error),
+ self.result(stdout=json.dumps(self.pages)),
+ self.result(stdout=json.dumps(release("v6.6.0"))),
+ ]
+ errors = io.StringIO()
+ with patch.object(versions.subprocess, "run", side_effect=results) as command, \
+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(errors):
+ releases, latest = versions.discover("rcourtman/Pulse")
+ self.assertEqual([self.inventory_command, self.inventory_command, self.latest_command],
+ [call.args[0] for call in command.call_args_list])
+ self.assertTrue(all(call.kwargs["timeout"] == 60 for call in command.call_args_list))
+ sleep.assert_called_once_with(2)
+ self.assertIn(self.stream_error, errors.getvalue())
+ self.assertEqual(versions.resolve(releases, latest).to_tag, "v6.7.0-beta.1")
+
+ def test_latest_read_retries_without_reloading_inventory(self):
+ results = [self.result(stdout=json.dumps(self.pages)),
+ self.result(1, "partial latest", self.stream_error),
+ self.result(stdout=json.dumps(release("v6.6.0")))]
+ with patch.object(versions.subprocess, "run", side_effect=results) as command, \
+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(io.StringIO()):
+ self.assertEqual(versions.discover("rcourtman/Pulse")[1], "v6.6.0")
+ self.assertEqual([self.inventory_command, self.latest_command, self.latest_command],
+ [call.args[0] for call in command.call_args_list])
+ sleep.assert_called_once_with(2)
+
+ def test_retry_budget_exhaustion_emits_no_pair(self):
+ output, errors = io.StringIO(), io.StringIO()
+ with patch.object(versions.subprocess, "run", return_value=self.result(1, "[]", self.stream_error)) as command, \
+ patch.object(versions.time, "sleep") as sleep, \
+ contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors):
+ self.assertEqual(versions.main(["--repository", "rcourtman/Pulse"]), 1)
+ self.assertEqual(command.call_count, 3)
+ self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 4])
+ self.assertEqual(errors.getvalue().count(self.stream_error), 3)
+ self.assertIn("attempt 3/3", errors.getvalue())
+ self.assertEqual(output.getvalue(), "")
+
+ def test_access_unknown_and_mixed_errors_never_retry(self):
+ for detail in ("HTTP 401", "HTTP 403", "HTTP 404", "HTTP 429", "HTTP 503",
+ "capability refused", "connection refused", "", "unavailable",
+ self.stream_error.replace("CANCEL", "REFUSED_STREAM")):
+ for error in (detail, self.stream_error + detail + "\n"):
+ with self.subTest(error=error), \
+ patch.object(versions.subprocess, "run", return_value=self.result(1, "[]", error)) as command, \
+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(io.StringIO()):
+ with self.assertRaises(versions.ResolutionError):
+ versions.discover("rcourtman/Pulse")
+ self.assertEqual(command.call_count, 1)
+ sleep.assert_not_called()
+
+ def test_signal_cancellation_and_other_exits_never_retry(self):
+ for status in (-15, -2, 2, 124, 130, 137, 143):
+ with self.subTest(status=status), \
+ patch.object(versions.subprocess, "run", return_value=self.result(status, "[]", self.stream_error)) as command, \
+ patch.object(versions.time, "sleep") as sleep, contextlib.redirect_stderr(io.StringIO()):
+ with self.assertRaisesRegex(versions.ResolutionError, f"exit {status}"):
+ versions.discover("rcourtman/Pulse")
+ self.assertEqual(command.call_count, 1)
+ sleep.assert_not_called()
+
+ def test_timeout_never_retries(self):
+ with patch.object(versions.subprocess, "run", side_effect=subprocess.TimeoutExpired("gh", 60)) as command, \
+ patch.object(versions.time, "sleep") as sleep:
+ with self.assertRaisesRegex(versions.ResolutionError, "timed out; not retrying"):
+ versions.discover("rcourtman/Pulse")
+ self.assertEqual(command.call_count, 1)
+ sleep.assert_not_called()
+
+ def test_malformed_inventory_stops_before_latest_read(self):
+ for payload in ("not json", "{}", "[]", '[{"tag_name":"v6.6.0"}]', '[[],{}]', '[[null]]'):
+ with self.subTest(payload=payload), \
+ patch.object(versions.subprocess, "run", return_value=self.result(stdout=payload)) as command, \
+ patch.object(versions.time, "sleep") as sleep:
+ with self.assertRaises((versions.ResolutionError, json.JSONDecodeError)):
+ versions.discover("rcourtman/Pulse")
+ self.assertEqual(command.call_count, 1)
+ sleep.assert_not_called()
+
+ def test_malformed_latest_stops_without_retry(self):
+ for payload in ("not json", "[]", "{}", '{"tag_name":null}'):
+ with self.subTest(payload=payload), \
+ patch.object(versions.subprocess, "run", side_effect=[
+ self.result(stdout=json.dumps(self.pages)), self.result(stdout=payload)]) as command, \
+ patch.object(versions.time, "sleep") as sleep:
+ with self.assertRaises((versions.ResolutionError, json.JSONDecodeError)):
+ versions.discover("rcourtman/Pulse")
+ self.assertEqual(command.call_count, 2)
+ sleep.assert_not_called()
+
+ def test_invalid_repository_and_live_latest_override_do_not_start_reads(self):
+ with patch.object(versions.subprocess, "run") as command, \
+ contextlib.redirect_stderr(io.StringIO()):
+ for repository in ("../Pulse", "rcourtman/Pulse/other", "https://example.com", "-H", "rcourtman/Pulse\n"):
+ with self.subTest(repository=repository), self.assertRaises(versions.ResolutionError):
+ versions.discover(repository)
+ with self.assertRaises(SystemExit) as error:
+ versions.main(["--repository", "rcourtman/Pulse", "--latest-tag", "v6.5.0"])
+ self.assertEqual(error.exception.code, 2)
+ command.assert_not_called()
+
+ def workflow_resolution(self, outcomes, workflow=None):
+ # Execute the actual workflow shell and Python CLI, replacing only gh
+ # with a process fixture. No installation, network or token is used.
+ if workflow is None:
+ workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
+ step = workflow.split(" - name: Resolve published release pair\n", 1)[1].split("\n - name:", 1)[0]
+ body = textwrap.dedent(step.split(" run: |\n", 1)[1])
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ (root / "outcomes.json").write_text(json.dumps(outcomes))
+ gh = root / "gh"
+ gh.write_text(f"#!{sys.executable}\n" + '''import json, os, pathlib, sys
+root = pathlib.Path(os.environ['DISCOVERY_FIXTURE'])
+with (root / 'calls.jsonl').open('a') as out:
+ out.write(json.dumps(sys.argv[1:]) + '\\n')
+count = len((root / 'calls.jsonl').read_text().splitlines())
+outcomes = json.loads((root / 'outcomes.json').read_text())
+status, output, error = outcomes[min(count - 1, len(outcomes) - 1)]
+print(output, end='', file=sys.stdout)
+print(error, end='', file=sys.stderr)
+sys.exit(status)
+''')
+ gh.chmod(0o755)
+ output, summary = root / "outputs", root / "summary"
+ result = subprocess.run(["bash", "-c", body], cwd=REPO_ROOT, capture_output=True,
+ text=True, timeout=15, env={**os.environ,
+ "PATH": str(root) + os.pathsep + os.environ["PATH"],
+ "DISCOVERY_FIXTURE": str(root), "GH_TOKEN": "synthetic",
+ "REPOSITORY": "rcourtman/Pulse", "FROM_INPUT": "", "TO_INPUT": "",
+ "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(summary)})
+ calls = [json.loads(line) for line in (root / "calls.jsonl").read_text().splitlines()]
+ return (result, calls, output.read_text() if output.exists() else "",
+ summary.read_text() if summary.exists() else "")
+
+ def test_workflow_recovers_only_metadata_and_writes_real_pair_outputs(self):
+ result, calls, output, summary = self.workflow_resolution([
+ (1, json.dumps([[release("v99.0.0")]]), self.stream_error),
+ (0, json.dumps(self.pages), ""), (0, json.dumps(release("v6.6.0")), ""),
+ ])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual([self.inventory_command[1:], self.inventory_command[1:], self.latest_command[1:]], calls)
+ self.assertRegex(output, r"from_tag<<[^\n]+\nv6\.6\.0\n")
+ self.assertRegex(output, r"to_tag<<[^\n]+\nv6\.7\.0-beta\.1\n")
+ self.assertNotIn("v99.0.0", output + summary)
+ self.assertIn("v6.7.0-beta.1", summary)
+ self.assertIn(self.stream_error, result.stderr)
+
+ def test_workflow_refusal_and_malformed_metadata_emit_no_outputs(self):
+ for outcome in ((1, "[]", "gh: Forbidden (HTTP 403)\n"), (0, "not json", "")):
+ with self.subTest(outcome=outcome):
+ result, calls, output, summary = self.workflow_resolution([outcome])
+ self.assertEqual(result.returncode, 1, result.stderr)
+ self.assertEqual(calls, [self.inventory_command[1:]])
+ self.assertEqual(output + summary, "")
+
+
class HarnessContractTest(unittest.TestCase):
harness = HARNESS_PATH.read_text(encoding="utf-8")