Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Fence delayed stable patches at reservation and publication

pulse-triage[bot]aa30322890c6
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
index 7912e97807..a6dd7ac739 100644
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@@ -2158,6 +2158,10 @@ jobs:
GH_TOKEN="${IMMUTABILITY_ADMIN_TOKEN}" \
./scripts/check-github-release-immutability.sh "${GITHUB_REPOSITORY}"
unset IMMUTABILITY_ADMIN_TOKEN
+ # Builds and staging may outlive patch eligibility or its stable base.
+ # Re-read both immediately before the irreversible publication call.
+ python3 scripts/release_control/stable_patch.py activation-check \
+ --version "${TAG}" --source-sha "${EXPECTED_COMMIT}"
gh api "repos/${{ github.repository }}/releases/${RELEASE_ID}" \
-X PATCH --input "$publish_payload" > "$release_json"
activated=true
diff --git a/.github/workflows/recover-release-activation.yml b/.github/workflows/recover-release-activation.yml
index e66ec67264..44bca1b27e 100644
--- a/.github/workflows/recover-release-activation.yml
+++ b/.github/workflows/recover-release-activation.yml
@@ -502,6 +502,9 @@ jobs:
GH_TOKEN="${IMMUTABILITY_ADMIN_TOKEN}" \
./scripts/check-github-release-immutability.sh "${GITHUB_REPOSITORY}"
unset IMMUTABILITY_ADMIN_TOKEN
+ # Activation-only recovery cannot publish an obsolete or late patch.
+ python3 scripts/release_control/stable_patch.py activation-check \
+ --version "${TAG}" --source-sha "${EXPECTED_COMMIT}"
gh api "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \
-X PATCH --input "${publish_payload}" > "${release_json}"
activated=true
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index cc416e91e4..ebc7387716 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -46,6 +46,22 @@ with its existing read-only permission and public checks. Private credentialed
governance remains post-merge only. A missing required context stays pending in
the train; branch admission does not remove or synthesise a required check.
+Slow collection cannot reserve an obsolete base or consume fixes after the
+48-hour cut boundary. The train rechecks the current stable's tag/source and
+minor-anchored clock at reservation and before later preparation, merge and
+dispatch mutations. A superseded/late reserved patch is abandoned without
+changing the independent minor's state.
+
+Both normal activation and activation-only recovery run the source-bound
+`stable_patch.py activation-check` immediately before their publication PATCH.
+It reads the exact candidate manifest, complete bounded published-release
+inventory, base-tag commit and the minor's actual publication date. A newer
+stable, changed base, unknown/failed evidence or arrival at the 48-hour boundary
+stops publication. A recent patch cannot reset the minor clock. This is a final
+read-before-write fence, not an atomic cross-workflow provider transaction;
+existing activation identities, quarantine and immutability checks remain.
+Minor/beta and historical-line paths gain no new patch gate.
+
### Unraid boot replacement preserves operator work
Agent setup/upgrade replaces only complete exact generated top-level go entries.
diff --git a/scripts/release_control/stable_patch.py b/scripts/release_control/stable_patch.py
index 722fadaece..4b5f824bd2 100644
--- a/scripts/release_control/stable_patch.py
+++ b/scripts/release_control/stable_patch.py
@@ -7,12 +7,14 @@ selection again and admits only release metadata on top, never other main work.
from __future__ import annotations
import argparse
+from datetime import datetime, timezone
import json
from pathlib import Path
import re
import subprocess
import tempfile
import sys
+import time
sys.dont_write_bytecode = True
@@ -20,6 +22,12 @@ SHA = re.compile(r"[0-9a-f]{40}")
VERSION = re.compile(r"(\d+)\.(\d+)\.([1-9]\d*)")
CONTROLS = (".github/workflows", "scripts/release_control")
SCHEMA = 1
+# Founder direction, 14/26 October: weekly minor, 24-hour candidate soak.
+# This publication fence has no caller-selected cadence or clock override.
+MINOR_CYCLE_SECONDS = 7 * 86400
+MINOR_SOAK_SECONDS = 24 * 3600
+PATCH_MARGIN_SECONDS = 48 * 3600
+REPOSITORY = "rcourtman/Pulse"
def git(root: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess:
@@ -163,16 +171,103 @@ def validate(root: Path, version: str, source_branch: str) -> dict:
return receipt
+def api(path: str) -> object:
+ """Read fixed project evidence only; a failed read is never retried or defaulted."""
+ result = subprocess.run(["gh", "api", f"repos/{REPOSITORY}/{path}"],
+ check=True, text=True, capture_output=True)
+ return json.loads(result.stdout)
+
+
+def published_at(row: dict, tag: str) -> float:
+ if (row.get("tag_name") != tag or row.get("draft") is not False
+ or row.get("prerelease") is not False or not row.get("published_at")):
+ raise ValueError("patch base/anchor must be a published stable release")
+ stamp = datetime.fromisoformat(row["published_at"].replace("Z", "+00:00"))
+ if stamp.tzinfo is None:
+ raise ValueError("stable publication time has no timezone")
+ return stamp.timestamp()
+
+
+def activation_check(root: Path, version: str, source_sha: str) -> dict:
+ """Re-read current stable and the clock immediately before public activation.
+
+ Source reconstruction earlier in the pipeline cannot establish that its
+ stable base is still current after a long build or activation-only recovery.
+ Minor, beta and historical-line publication rules are deliberately unchanged.
+ """
+ match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:-([a-z]+)\.(\d+))?", version)
+ if not match:
+ raise ValueError("invalid release version")
+ major, minor, patch = map(int, match.groups()[:3])
+ if not patch or (major, minor) < (6, 5):
+ return {"applicable": False}
+ if match.group(4) and match.groups()[3:] != ("rc", "1"):
+ raise ValueError("mechanical patches permit only rc.1 and stable")
+ if not SHA.fullmatch(source_sha):
+ raise ValueError("activation source must be an exact SHA")
+ core = f"{major}.{minor}.{patch}"
+ receipt = json.loads(value(root, "show", f"{source_sha}:{manifest_path(core)}"))
+ if receipt["version"] != core or receipt["stable_tag"] != f"v{major}.{minor}.{patch - 1}":
+ raise ValueError("activation manifest does not identify the preceding stable")
+ if not SHA.fullmatch(receipt["stable_sha"]):
+ raise ValueError("activation manifest has no exact stable source")
+ # Pagination is bounded and complete. A short/truncated first page cannot
+ # hide a newer stable behind a run of previews or draft releases.
+ rows = []
+ for page in range(1, 101):
+ batch = api(f"releases?per_page=100&page={page}")
+ if not isinstance(batch, list):
+ raise ValueError("stable release inventory unavailable")
+ for row in batch:
+ if not isinstance(row, dict) or not isinstance(row.get("tag_name"), str):
+ raise ValueError("malformed release inventory")
+ if re.fullmatch(r"v\d+\.\d+\.\d+", row["tag_name"]):
+ if type(row.get("draft")) is not bool or type(row.get("prerelease")) is not bool:
+ raise ValueError("stable release inventory has unknown publication state")
+ rows.extend(batch)
+ if len(batch) < 100:
+ break
+ else:
+ raise ValueError("stable release inventory exceeds bounded pagination")
+ stable = [row for row in rows if row.get("draft") is False
+ and row.get("prerelease") is False
+ and re.fullmatch(r"v\d+\.\d+\.\d+", row.get("tag_name", ""))]
+ if not stable:
+ raise ValueError("no current published stable")
+ current = max(stable, key=lambda row: tuple(map(int, row["tag_name"][1:].split("."))))
+ if current["tag_name"] != receipt["stable_tag"]:
+ raise ValueError("a newer stable superseded the patch base before activation")
+ published_at(current, receipt["stable_tag"])
+ commit = api(f"commits/{receipt['stable_tag']}")
+ if not isinstance(commit, dict) or commit.get("sha") != receipt["stable_sha"]:
+ raise ValueError("published patch base source changed before activation")
+ anchor_tag = f"v{major}.{minor}.0"
+ anchor = current if current["tag_name"] == anchor_tag else api(f"releases/tags/{anchor_tag}")
+ due = published_at(anchor, anchor_tag) + MINOR_CYCLE_SECONDS - MINOR_SOAK_SECONDS
+ # Read the time after provider evidence, not before potentially slow reads.
+ observed = time.time()
+ if observed >= due - PATCH_MARGIN_SECONDS:
+ raise ValueError("patch activation is within 48 hours of minor candidate eligibility (or overdue)")
+ return {"applicable": True, "source_sha": source_sha, "stable_tag": current["tag_name"],
+ "stable_sha": receipt["stable_sha"], "observed_at": observed,
+ "not_after": datetime.fromtimestamp(due - PATCH_MARGIN_SECONDS, timezone.utc).isoformat()}
+
+
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("operation", choices=("assemble", "validate"))
+ parser.add_argument("operation", choices=("assemble", "validate", "activation-check"))
parser.add_argument("--selection", type=Path)
parser.add_argument("--version")
parser.add_argument("--source-branch")
+ parser.add_argument("--source-sha")
args = parser.parse_args()
root = Path.cwd().resolve()
- result = assemble(root, json.loads(args.selection.read_text())) if args.operation == "assemble" \
- else validate(root, args.version, args.source_branch)
+ if args.operation == "assemble":
+ result = assemble(root, json.loads(args.selection.read_text()))
+ elif args.operation == "validate":
+ result = validate(root, args.version, args.source_branch)
+ else:
+ result = activation_check(root, args.version, args.source_sha)
print(json.dumps(result, sort_keys=True))
diff --git a/scripts/release_control/stable_patch_test.py b/scripts/release_control/stable_patch_test.py
index 593136b412..565fb81972 100644
--- a/scripts/release_control/stable_patch_test.py
+++ b/scripts/release_control/stable_patch_test.py
@@ -1,7 +1,9 @@
"""Whole merge results and exact controls, with real Git but no network or publishing."""
from copy import deepcopy
+from datetime import datetime, timezone
from fnmatch import fnmatchcase
import json
+import os
from pathlib import Path
import subprocess
import tempfile
@@ -166,6 +168,156 @@ class StablePatchTest(unittest.TestCase):
self.assemble()
self.assertEqual('do not discard', (self.root / 'untracked').read_text())
+ def activation_evidence(self, *, tag='v6.6.0', age=86400):
+ now = 1800000000
+ row = {'tag_name': tag, 'draft': False, 'prerelease': False,
+ 'published_at': datetime.fromtimestamp(now - age, timezone.utc).isoformat()}
+ return now, row
+
+ def test_activation_rechecks_exact_base_and_late_clock_for_rc_and_stable(self):
+ self.assemble()
+ stamp, stable = self.activation_evidence()
+ for version in ('v6.6.1-rc.1', 'v6.6.1'):
+ for elapsed in (0, 3 * 86400 - 1, 3 * 86400, 9 * 86400):
+ def read(path):
+ # The clock can advance while API collection is in flight.
+ clock.return_value = stamp + elapsed
+ if path == 'releases?per_page=100&page=1':
+ return [stable]
+ self.assertEqual('commits/v6.6.0', path)
+ return {'sha': self.base}
+ with self.subTest(version=version, elapsed=elapsed), \
+ mock.patch.object(patch, 'api', side_effect=read), \
+ mock.patch.object(patch.time, 'time', return_value=stamp) as clock:
+ if elapsed >= 3 * 86400:
+ with self.assertRaisesRegex(ValueError, '48 hours'):
+ patch.activation_check(self.root, version, self.head())
+ else:
+ result = patch.activation_check(self.root, version, self.head())
+ self.assertEqual((True, self.base, stamp + elapsed),
+ (result['applicable'], result['stable_sha'], result['observed_at']))
+
+ def test_new_minor_and_base_tag_drift_stop_activation_after_source_admission(self):
+ self.assemble()
+ # A valid reconstruction is not a durable publication permission.
+ patch.validate(self.root, '6.6.1-rc.1', 'release/v6.6.1')
+ stamp, stable = self.activation_evidence()
+ for rows, sha, message in (([stable, {**stable, 'tag_name': 'v6.7.0'}], self.base, 'newer stable'),
+ ([stable], self.fix, 'source changed')):
+ with self.subTest(message=message), mock.patch.object(patch, 'api', side_effect=[rows, {'sha': sha}]), \
+ mock.patch.object(patch.time, 'time', return_value=stamp), self.assertRaisesRegex(ValueError, message):
+ patch.activation_check(self.root, '6.6.1', self.head())
+
+ def test_activation_has_no_minor_beta_or_historical_patch_gate(self):
+ with mock.patch.object(patch, 'api', side_effect=AssertionError('no patch API read')):
+ for version in ('6.7.0', 'v6.7.0-rc.1', 'v6.7.0-beta.1', 'v6.4.5', 'v6.4.5-rc.6'):
+ self.assertEqual({'applicable': False}, patch.activation_check(self.root, version, ''))
+
+ def test_activation_cannot_use_missing_or_other_source_manifest_or_second_rc(self):
+ self.assemble()
+ with mock.patch.object(patch, 'api', side_effect=AssertionError('local binding first')):
+ for version, sha in (('6.6.1', self.base), ('6.6.2', self.head()),
+ ('6.6.1-rc.2', self.head()), ('6.6.1-beta.1', self.head()), ('6.6.1', 'main')):
+ with self.subTest(version=version, sha=sha), self.assertRaises((ValueError, subprocess.CalledProcessError)):
+ patch.activation_check(self.root, version, sha)
+
+ def test_inventory_and_anchor_failures_never_turn_into_permission(self):
+ self.assemble()
+ stamp, stable = self.activation_evidence()
+ malformed = [None, {}, [None], [{'tag_name': 'v6.7.0'}], [],
+ [{**stable, 'published_at': '2026-10-27T10:00:00'}]]
+ for rows in malformed:
+ with self.subTest(rows=rows), mock.patch.object(patch, 'api', return_value=rows), \
+ mock.patch.object(patch.time, 'time', return_value=stamp), self.assertRaises((ValueError, TypeError)):
+ patch.activation_check(self.root, '6.6.1', self.head())
+ with mock.patch.object(patch, 'api', side_effect=subprocess.CalledProcessError(1, 'gh')) as api, \
+ self.assertRaises(subprocess.CalledProcessError):
+ patch.activation_check(self.root, '6.6.1', self.head())
+ api.assert_called_once() # no fallback/retry after a failed evidence read
+
+ def test_pagination_cannot_hide_newer_stable_behind_previews(self):
+ self.assemble()
+ _, stable = self.activation_evidence()
+ preview = {**stable, 'prerelease': True, 'tag_name': 'v6.7.0-beta.1'}
+ with mock.patch.object(patch, 'api', side_effect=[[preview] * 100,
+ [stable, {**stable, 'tag_name': 'v6.7.0'}]]) as api, \
+ self.assertRaisesRegex(ValueError, 'newer stable'):
+ patch.activation_check(self.root, '6.6.1', self.head())
+ self.assertEqual('releases?per_page=100&page=2', api.call_args.args[0])
+
+ def test_later_patch_uses_minor_publication_not_recent_patch_clock(self):
+ self.assemble()
+ self.git('tag', 'v6.6.1')
+ prior = self.head()
+ self.git('checkout', '--quiet', 'main')
+ self.write('later', 'next fix')
+ self.commit('next main fix')
+ merge = self.head()
+ self.selection.update(version='6.6.2', stable_tag='v6.6.1', stable_sha=prior,
+ control_sha=merge, fixes=[{'number': 15, 'merge_sha': merge, 'issues': [2]}])
+ self.assemble()
+ stamp, stable = self.activation_evidence(tag='v6.6.1', age=60)
+ _, anchor = self.activation_evidence(age=4 * 86400)
+ for observed_anchor in (anchor, {**anchor, 'draft': True}):
+ with mock.patch.object(patch, 'api', side_effect=[[stable], {'sha': prior}, observed_anchor]) as api, \
+ mock.patch.object(patch.time, 'time', return_value=stamp), self.assertRaises(ValueError):
+ patch.activation_check(self.root, '6.6.2', self.head())
+ self.assertEqual('releases/tags/v6.6.0', api.call_args.args[0])
+
+ def test_actual_activation_and_recovery_shell_refuse_before_publication_post(self):
+ """Execute the final workflow boundary with the real guard and offline gh."""
+ import yaml
+ self.assemble()
+ repo = Path(__file__).resolve().parents[2]
+ bin_dir = Path(self.temp.name) / 'bin'
+ bin_dir.mkdir()
+ gh = bin_dir / 'gh'
+ gh.write_text('''#!/usr/bin/env python3
+import json,os,pathlib,sys
+args=sys.argv[1:]
+if '-X' in args:
+ assert args[args.index('-X')+1] == 'PATCH'
+ pathlib.Path(os.environ['POST_MARKER']).write_text('publication attempted')
+ print('{}')
+else:
+ assert args[0] == 'api'
+ evidence=json.loads(pathlib.Path(os.environ['API_EVIDENCE']).read_text())
+ if args[1] not in evidence: sys.exit(22)
+ print(json.dumps(evidence[args[1]]))
+''')
+ gh.chmod(0o755)
+ evidence = Path(self.temp.name) / 'api.json'
+ marker = Path(self.temp.name) / 'post'
+ now = __import__('time').time()
+ _, stable = self.activation_evidence()
+ stable['published_at'] = datetime.fromtimestamp(now - 86400, timezone.utc).isoformat()
+ env = {**os.environ, 'PATH': str(bin_dir) + os.pathsep + os.environ['PATH'],
+ 'TAG': 'v6.6.1', 'EXPECTED_COMMIT': self.head(), 'RELEASE_ID': '123',
+ 'GITHUB_REPOSITORY': patch.REPOSITORY, 'POST_MARKER': str(marker),
+ 'release_json': str(Path(self.temp.name) / 'published.json'),
+ 'publish_payload': str(Path(self.temp.name) / 'payload.json'),
+ 'API_EVIDENCE': str(evidence)}
+ for path, job_name, step_name in (('.github/workflows/create-release.yml', 'activate_release',
+ 'Publish the fully staged release'),
+ ('.github/workflows/recover-release-activation.yml', 'recover_activation',
+ 'Commit the recovered activation')):
+ workflow = yaml.load((repo / path).read_text(), Loader=yaml.BaseLoader)
+ script = next(step['run'] for step in workflow['jobs'][job_name]['steps'] if step.get('name') == step_name)
+ boundary = script.split('unset IMMUTABILITY_ADMIN_TOKEN\n', 1)[1].split('activated=true', 1)[0]
+ self.assertIn('stable_patch.py activation-check', boundary)
+ boundary = boundary.replace('python3 scripts/release_control/stable_patch.py',
+ 'python3 ' + str(repo / 'scripts/release_control/stable_patch.py'))
+ boundary = boundary.replace('${{ github.repository }}', '${GITHUB_REPOSITORY}')
+ for current in (stable, {**stable, 'tag_name': 'v6.7.0'}):
+ marker.unlink(missing_ok=True)
+ evidence.write_text(json.dumps({f'repos/{patch.REPOSITORY}/releases?per_page=100&page=1': [current],
+ f'repos/{patch.REPOSITORY}/commits/v6.6.0': {'sha': self.base}}))
+ result = subprocess.run(['bash', '-euc', boundary], cwd=self.root, env=env,
+ capture_output=True, text=True)
+ with self.subTest(workflow=path, current=current['tag_name']):
+ self.assertEqual(current is stable, result.returncode == 0, result.stderr)
+ self.assertEqual(current is stable, marker.exists())
+
class StablePatchGovernanceTest(unittest.TestCase):
"""Real patch targets must produce the independently required check context."""
diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py
index 1b04031eb5..13c717c3a7 100644
--- a/scripts/release_control/subsystem_lookup_test.py
+++ b/scripts/release_control/subsystem_lookup_test.py
@@ -65,6 +65,7 @@ RELEASE_PROMOTION_METADATA_EXACT_FILES = [
"scripts/release_control/release_promotion_policy_test.py",
"scripts/release_control/render_release_body_test.py",
"scripts/release_control/resolve_release_promotion_test.py",
+ "scripts/release_control/stable_patch_test.py",
"scripts/release_control/validate_artifact_release_line_test.py",
"scripts/release_control/verify_github_release_integrity_test.py",
]