Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Fence delayed stable patches at reservation and publication
pulse-triage[bot]aa30322890c6
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.ymlindex 7912e97807..a6dd7ac739 100644--- a/.github/workflows/create-release.yml+++ b/.github/workflows/create-release.yml@@ -2158,6 +2158,10 @@ jobs:GH_TOKEN="${IMMUTABILITY_ADMIN_TOKEN}" \./scripts/check-github-release-immutability.sh "${GITHUB_REPOSITORY}"unset IMMUTABILITY_ADMIN_TOKEN+ # Builds and staging may outlive patch eligibility or its stable base.+ # Re-read both immediately before the irreversible publication call.+ python3 scripts/release_control/stable_patch.py activation-check \+ --version "${TAG}" --source-sha "${EXPECTED_COMMIT}"gh api "repos/${{ github.repository }}/releases/${RELEASE_ID}" \-X PATCH --input "$publish_payload" > "$release_json"activated=truediff --git a/.github/workflows/recover-release-activation.yml b/.github/workflows/recover-release-activation.ymlindex e66ec67264..44bca1b27e 100644--- a/.github/workflows/recover-release-activation.yml+++ b/.github/workflows/recover-release-activation.yml@@ -502,6 +502,9 @@ jobs:GH_TOKEN="${IMMUTABILITY_ADMIN_TOKEN}" \./scripts/check-github-release-immutability.sh "${GITHUB_REPOSITORY}"unset IMMUTABILITY_ADMIN_TOKEN+ # Activation-only recovery cannot publish an obsolete or late patch.+ python3 scripts/release_control/stable_patch.py activation-check \+ --version "${TAG}" --source-sha "${EXPECTED_COMMIT}"gh api "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \-X PATCH --input "${publish_payload}" > "${release_json}"activated=truediff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex cc416e91e4..ebc7387716 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -46,6 +46,22 @@ with its existing read-only permission and public checks. Private credentialedgovernance remains post-merge only. A missing required context stays pending inthe train; branch admission does not remove or synthesise a required check.+Slow collection cannot reserve an obsolete base or consume fixes after the+48-hour cut boundary. The train rechecks the current stable's tag/source and+minor-anchored clock at reservation and before later preparation, merge and+dispatch mutations. A superseded/late reserved patch is abandoned without+changing the independent minor's state.++Both normal activation and activation-only recovery run the source-bound+`stable_patch.py activation-check` immediately before their publication PATCH.+It reads the exact candidate manifest, complete bounded published-release+inventory, base-tag commit and the minor's actual publication date. A newer+stable, changed base, unknown/failed evidence or arrival at the 48-hour boundary+stops publication. A recent patch cannot reset the minor clock. This is a final+read-before-write fence, not an atomic cross-workflow provider transaction;+existing activation identities, quarantine and immutability checks remain.+Minor/beta and historical-line paths gain no new patch gate.+### Unraid boot replacement preserves operator workAgent setup/upgrade replaces only complete exact generated top-level go entries.diff --git a/scripts/release_control/stable_patch.py b/scripts/release_control/stable_patch.pyindex 722fadaece..4b5f824bd2 100644--- a/scripts/release_control/stable_patch.py+++ b/scripts/release_control/stable_patch.py@@ -7,12 +7,14 @@ selection again and admits only release metadata on top, never other main work.from __future__ import annotationsimport argparse+from datetime import datetime, timezoneimport jsonfrom pathlib import Pathimport reimport subprocessimport tempfileimport sys+import timesys.dont_write_bytecode = True@@ -20,6 +22,12 @@ SHA = re.compile(r"[0-9a-f]{40}")VERSION = re.compile(r"(\d+)\.(\d+)\.([1-9]\d*)")CONTROLS = (".github/workflows", "scripts/release_control")SCHEMA = 1+# Founder direction, 14/26 October: weekly minor, 24-hour candidate soak.+# This publication fence has no caller-selected cadence or clock override.+MINOR_CYCLE_SECONDS = 7 * 86400+MINOR_SOAK_SECONDS = 24 * 3600+PATCH_MARGIN_SECONDS = 48 * 3600+REPOSITORY = "rcourtman/Pulse"def git(root: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess:@@ -163,16 +171,103 @@ def validate(root: Path, version: str, source_branch: str) -> dict:return receipt+def api(path: str) -> object:+ """Read fixed project evidence only; a failed read is never retried or defaulted."""+ result = subprocess.run(["gh", "api", f"repos/{REPOSITORY}/{path}"],+ check=True, text=True, capture_output=True)+ return json.loads(result.stdout)+++def published_at(row: dict, tag: str) -> float:+ if (row.get("tag_name") != tag or row.get("draft") is not False+ or row.get("prerelease") is not False or not row.get("published_at")):+ raise ValueError("patch base/anchor must be a published stable release")+ stamp = datetime.fromisoformat(row["published_at"].replace("Z", "+00:00"))+ if stamp.tzinfo is None:+ raise ValueError("stable publication time has no timezone")+ return stamp.timestamp()+++def activation_check(root: Path, version: str, source_sha: str) -> dict:+ """Re-read current stable and the clock immediately before public activation.++ Source reconstruction earlier in the pipeline cannot establish that its+ stable base is still current after a long build or activation-only recovery.+ Minor, beta and historical-line publication rules are deliberately unchanged.+ """+ match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-([a-z]+)\.(\d+))?", version)+ if not match:+ raise ValueError("invalid release version")+ major, minor, patch = map(int, match.groups()[:3])+ if not patch or (major, minor) < (6, 5):+ return {"applicable": False}+ if match.group(4) and match.groups()[3:] != ("rc", "1"):+ raise ValueError("mechanical patches permit only rc.1 and stable")+ if not SHA.fullmatch(source_sha):+ raise ValueError("activation source must be an exact SHA")+ core = f"{major}.{minor}.{patch}"+ receipt = json.loads(value(root, "show", f"{source_sha}:{manifest_path(core)}"))+ if receipt["version"] != core or receipt["stable_tag"] != f"v{major}.{minor}.{patch - 1}":+ raise ValueError("activation manifest does not identify the preceding stable")+ if not SHA.fullmatch(receipt["stable_sha"]):+ raise ValueError("activation manifest has no exact stable source")+ # Pagination is bounded and complete. A short/truncated first page cannot+ # hide a newer stable behind a run of previews or draft releases.+ rows = []+ for page in range(1, 101):+ batch = api(f"releases?per_page=100&page={page}")+ if not isinstance(batch, list):+ raise ValueError("stable release inventory unavailable")+ for row in batch:+ if not isinstance(row, dict) or not isinstance(row.get("tag_name"), str):+ raise ValueError("malformed release inventory")+ if re.fullmatch(r"v\d+\.\d+\.\d+", row["tag_name"]):+ if type(row.get("draft")) is not bool or type(row.get("prerelease")) is not bool:+ raise ValueError("stable release inventory has unknown publication state")+ rows.extend(batch)+ if len(batch) < 100:+ break+ else:+ raise ValueError("stable release inventory exceeds bounded pagination")+ stable = [row for row in rows if row.get("draft") is False+ and row.get("prerelease") is False+ and re.fullmatch(r"v\d+\.\d+\.\d+", row.get("tag_name", ""))]+ if not stable:+ raise ValueError("no current published stable")+ current = max(stable, key=lambda row: tuple(map(int, row["tag_name"][1:].split("."))))+ if current["tag_name"] != receipt["stable_tag"]:+ raise ValueError("a newer stable superseded the patch base before activation")+ published_at(current, receipt["stable_tag"])+ commit = api(f"commits/{receipt['stable_tag']}")+ if not isinstance(commit, dict) or commit.get("sha") != receipt["stable_sha"]:+ raise ValueError("published patch base source changed before activation")+ anchor_tag = f"v{major}.{minor}.0"+ anchor = current if current["tag_name"] == anchor_tag else api(f"releases/tags/{anchor_tag}")+ due = published_at(anchor, anchor_tag) + MINOR_CYCLE_SECONDS - MINOR_SOAK_SECONDS+ # Read the time after provider evidence, not before potentially slow reads.+ observed = time.time()+ if observed >= due - PATCH_MARGIN_SECONDS:+ raise ValueError("patch activation is within 48 hours of minor candidate eligibility (or overdue)")+ return {"applicable": True, "source_sha": source_sha, "stable_tag": current["tag_name"],+ "stable_sha": receipt["stable_sha"], "observed_at": observed,+ "not_after": datetime.fromtimestamp(due - PATCH_MARGIN_SECONDS, timezone.utc).isoformat()}++def main() -> None:parser = argparse.ArgumentParser(description=__doc__)- parser.add_argument("operation", choices=("assemble", "validate"))+ parser.add_argument("operation", choices=("assemble", "validate", "activation-check"))parser.add_argument("--selection", type=Path)parser.add_argument("--version")parser.add_argument("--source-branch")+ parser.add_argument("--source-sha")args = parser.parse_args()root = Path.cwd().resolve()- result = assemble(root, json.loads(args.selection.read_text())) if args.operation == "assemble" \- else validate(root, args.version, args.source_branch)+ if args.operation == "assemble":+ result = assemble(root, json.loads(args.selection.read_text()))+ elif args.operation == "validate":+ result = validate(root, args.version, args.source_branch)+ else:+ result = activation_check(root, args.version, args.source_sha)print(json.dumps(result, sort_keys=True))diff --git a/scripts/release_control/stable_patch_test.py b/scripts/release_control/stable_patch_test.pyindex 593136b412..565fb81972 100644--- a/scripts/release_control/stable_patch_test.py+++ b/scripts/release_control/stable_patch_test.py@@ -1,7 +1,9 @@"""Whole merge results and exact controls, with real Git but no network or publishing."""from copy import deepcopy+from datetime import datetime, timezonefrom fnmatch import fnmatchcaseimport json+import osfrom pathlib import Pathimport subprocessimport tempfile@@ -166,6 +168,156 @@ class StablePatchTest(unittest.TestCase):self.assemble()self.assertEqual('do not discard', (self.root / 'untracked').read_text())+ def activation_evidence(self, *, tag='v6.6.0', age=86400):+ now = 1800000000+ row = {'tag_name': tag, 'draft': False, 'prerelease': False,+ 'published_at': datetime.fromtimestamp(now - age, timezone.utc).isoformat()}+ return now, row++ def test_activation_rechecks_exact_base_and_late_clock_for_rc_and_stable(self):+ self.assemble()+ stamp, stable = self.activation_evidence()+ for version in ('v6.6.1-rc.1', 'v6.6.1'):+ for elapsed in (0, 3 * 86400 - 1, 3 * 86400, 9 * 86400):+ def read(path):+ # The clock can advance while API collection is in flight.+ clock.return_value = stamp + elapsed+ if path == 'releases?per_page=100&page=1':+ return [stable]+ self.assertEqual('commits/v6.6.0', path)+ return {'sha': self.base}+ with self.subTest(version=version, elapsed=elapsed), \+ mock.patch.object(patch, 'api', side_effect=read), \+ mock.patch.object(patch.time, 'time', return_value=stamp) as clock:+ if elapsed >= 3 * 86400:+ with self.assertRaisesRegex(ValueError, '48 hours'):+ patch.activation_check(self.root, version, self.head())+ else:+ result = patch.activation_check(self.root, version, self.head())+ self.assertEqual((True, self.base, stamp + elapsed),+ (result['applicable'], result['stable_sha'], result['observed_at']))++ def test_new_minor_and_base_tag_drift_stop_activation_after_source_admission(self):+ self.assemble()+ # A valid reconstruction is not a durable publication permission.+ patch.validate(self.root, '6.6.1-rc.1', 'release/v6.6.1')+ stamp, stable = self.activation_evidence()+ for rows, sha, message in (([stable, {**stable, 'tag_name': 'v6.7.0'}], self.base, 'newer stable'),+ ([stable], self.fix, 'source changed')):+ with self.subTest(message=message), mock.patch.object(patch, 'api', side_effect=[rows, {'sha': sha}]), \+ mock.patch.object(patch.time, 'time', return_value=stamp), self.assertRaisesRegex(ValueError, message):+ patch.activation_check(self.root, '6.6.1', self.head())++ def test_activation_has_no_minor_beta_or_historical_patch_gate(self):+ with mock.patch.object(patch, 'api', side_effect=AssertionError('no patch API read')):+ for version in ('6.7.0', 'v6.7.0-rc.1', 'v6.7.0-beta.1', 'v6.4.5', 'v6.4.5-rc.6'):+ self.assertEqual({'applicable': False}, patch.activation_check(self.root, version, ''))++ def test_activation_cannot_use_missing_or_other_source_manifest_or_second_rc(self):+ self.assemble()+ with mock.patch.object(patch, 'api', side_effect=AssertionError('local binding first')):+ for version, sha in (('6.6.1', self.base), ('6.6.2', self.head()),+ ('6.6.1-rc.2', self.head()), ('6.6.1-beta.1', self.head()), ('6.6.1', 'main')):+ with self.subTest(version=version, sha=sha), self.assertRaises((ValueError, subprocess.CalledProcessError)):+ patch.activation_check(self.root, version, sha)++ def test_inventory_and_anchor_failures_never_turn_into_permission(self):+ self.assemble()+ stamp, stable = self.activation_evidence()+ malformed = [None, {}, [None], [{'tag_name': 'v6.7.0'}], [],+ [{**stable, 'published_at': '2026-10-27T10:00:00'}]]+ for rows in malformed:+ with self.subTest(rows=rows), mock.patch.object(patch, 'api', return_value=rows), \+ mock.patch.object(patch.time, 'time', return_value=stamp), self.assertRaises((ValueError, TypeError)):+ patch.activation_check(self.root, '6.6.1', self.head())+ with mock.patch.object(patch, 'api', side_effect=subprocess.CalledProcessError(1, 'gh')) as api, \+ self.assertRaises(subprocess.CalledProcessError):+ patch.activation_check(self.root, '6.6.1', self.head())+ api.assert_called_once() # no fallback/retry after a failed evidence read++ def test_pagination_cannot_hide_newer_stable_behind_previews(self):+ self.assemble()+ _, stable = self.activation_evidence()+ preview = {**stable, 'prerelease': True, 'tag_name': 'v6.7.0-beta.1'}+ with mock.patch.object(patch, 'api', side_effect=[[preview] * 100,+ [stable, {**stable, 'tag_name': 'v6.7.0'}]]) as api, \+ self.assertRaisesRegex(ValueError, 'newer stable'):+ patch.activation_check(self.root, '6.6.1', self.head())+ self.assertEqual('releases?per_page=100&page=2', api.call_args.args[0])++ def test_later_patch_uses_minor_publication_not_recent_patch_clock(self):+ self.assemble()+ self.git('tag', 'v6.6.1')+ prior = self.head()+ self.git('checkout', '--quiet', 'main')+ self.write('later', 'next fix')+ self.commit('next main fix')+ merge = self.head()+ self.selection.update(version='6.6.2', stable_tag='v6.6.1', stable_sha=prior,+ control_sha=merge, fixes=[{'number': 15, 'merge_sha': merge, 'issues': [2]}])+ self.assemble()+ stamp, stable = self.activation_evidence(tag='v6.6.1', age=60)+ _, anchor = self.activation_evidence(age=4 * 86400)+ for observed_anchor in (anchor, {**anchor, 'draft': True}):+ with mock.patch.object(patch, 'api', side_effect=[[stable], {'sha': prior}, observed_anchor]) as api, \+ mock.patch.object(patch.time, 'time', return_value=stamp), self.assertRaises(ValueError):+ patch.activation_check(self.root, '6.6.2', self.head())+ self.assertEqual('releases/tags/v6.6.0', api.call_args.args[0])++ def test_actual_activation_and_recovery_shell_refuse_before_publication_post(self):+ """Execute the final workflow boundary with the real guard and offline gh."""+ import yaml+ self.assemble()+ repo = Path(__file__).resolve().parents[2]+ bin_dir = Path(self.temp.name) / 'bin'+ bin_dir.mkdir()+ gh = bin_dir / 'gh'+ gh.write_text('''#!/usr/bin/env python3+import json,os,pathlib,sys+args=sys.argv[1:]+if '-X' in args:+ assert args[args.index('-X')+1] == 'PATCH'+ pathlib.Path(os.environ['POST_MARKER']).write_text('publication attempted')+ print('{}')+else:+ assert args[0] == 'api'+ evidence=json.loads(pathlib.Path(os.environ['API_EVIDENCE']).read_text())+ if args[1] not in evidence: sys.exit(22)+ print(json.dumps(evidence[args[1]]))+''')+ gh.chmod(0o755)+ evidence = Path(self.temp.name) / 'api.json'+ marker = Path(self.temp.name) / 'post'+ now = __import__('time').time()+ _, stable = self.activation_evidence()+ stable['published_at'] = datetime.fromtimestamp(now - 86400, timezone.utc).isoformat()+ env = {**os.environ, 'PATH': str(bin_dir) + os.pathsep + os.environ['PATH'],+ 'TAG': 'v6.6.1', 'EXPECTED_COMMIT': self.head(), 'RELEASE_ID': '123',+ 'GITHUB_REPOSITORY': patch.REPOSITORY, 'POST_MARKER': str(marker),+ 'release_json': str(Path(self.temp.name) / 'published.json'),+ 'publish_payload': str(Path(self.temp.name) / 'payload.json'),+ 'API_EVIDENCE': str(evidence)}+ for path, job_name, step_name in (('.github/workflows/create-release.yml', 'activate_release',+ 'Publish the fully staged release'),+ ('.github/workflows/recover-release-activation.yml', 'recover_activation',+ 'Commit the recovered activation')):+ workflow = yaml.load((repo / path).read_text(), Loader=yaml.BaseLoader)+ script = next(step['run'] for step in workflow['jobs'][job_name]['steps'] if step.get('name') == step_name)+ boundary = script.split('unset IMMUTABILITY_ADMIN_TOKEN\n', 1)[1].split('activated=true', 1)[0]+ self.assertIn('stable_patch.py activation-check', boundary)+ boundary = boundary.replace('python3 scripts/release_control/stable_patch.py',+ 'python3 ' + str(repo / 'scripts/release_control/stable_patch.py'))+ boundary = boundary.replace('${{ github.repository }}', '${GITHUB_REPOSITORY}')+ for current in (stable, {**stable, 'tag_name': 'v6.7.0'}):+ marker.unlink(missing_ok=True)+ evidence.write_text(json.dumps({f'repos/{patch.REPOSITORY}/releases?per_page=100&page=1': [current],+ f'repos/{patch.REPOSITORY}/commits/v6.6.0': {'sha': self.base}}))+ result = subprocess.run(['bash', '-euc', boundary], cwd=self.root, env=env,+ capture_output=True, text=True)+ with self.subTest(workflow=path, current=current['tag_name']):+ self.assertEqual(current is stable, result.returncode == 0, result.stderr)+ self.assertEqual(current is stable, marker.exists())+class StablePatchGovernanceTest(unittest.TestCase):"""Real patch targets must produce the independently required check context."""diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.pyindex 1b04031eb5..13c717c3a7 100644--- a/scripts/release_control/subsystem_lookup_test.py+++ b/scripts/release_control/subsystem_lookup_test.py@@ -65,6 +65,7 @@ RELEASE_PROMOTION_METADATA_EXACT_FILES = ["scripts/release_control/release_promotion_policy_test.py","scripts/release_control/render_release_body_test.py","scripts/release_control/resolve_release_promotion_test.py",+ "scripts/release_control/stable_patch_test.py","scripts/release_control/validate_artifact_release_line_test.py","scripts/release_control/verify_github_release_integrity_test.py",]