Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Release completed backend slice slots to ordinary packages
pulse-triage[bot]95811c800ac0
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.ymlindex c3b1c02667..c050f54f6c 100644--- a/.github/workflows/build-and-test.yml+++ b/.github/workflows/build-and-test.yml@@ -511,8 +511,8 @@ jobs:if: needs.changes.outputs.code == 'true'env:PULSE_DATA_DIR: /tmp/pulse-test-data- # Two ordinary package binaries plus the two sliced binaries: keep- # the four-package concurrency envelope, rather than adding runners.+ # Share four package slots; a finished slice releases its slot to+ # ordinary packages. Keep the same per-binary CPU bound and runners.GOMAXPROCS: '2'# Source, not the timing file, defines coverage. Longest measured# packages go to the lightest shard and start first within that shard;@@ -534,28 +534,13 @@ jobs:| grep -vE '/(internal/monitoring|scripts/installtests)$' \| python3 .github/scripts/select-go-package-shard.py \.github/scripts/go-package-test-seconds.txt 2 "$index")- if [ -z "$pkgs" ]; then- echo "shard resolved to an empty package list" >&2- exit 1- fi- printf 'Packages in this shard:\n%s\n' "$pkgs"- # GOMAXPROCS=2 bounds ordinary go test to two package workers. The- # other two binaries each run this worker's contiguous source-listed- # half. Preserve both protected rest names and compilation gates.- pids=()- go test -race -timeout 50m $pkgs &- pids+=("$!")- for package in internal/monitoring scripts/installtests; do- python3 scripts/release_control/run_go_package_slice.py \- --package "$package" --index "$index" \- --output "$RUNNER_TEMP/backend-slices/${package##*/}" &- pids+=("$!")- done- status=0- for pid in "${pids[@]}"; do- if wait "$pid"; then :; else status=1; fi- done- exit "$status"+ # Keep the stable-shipped command contract as the actual template,+ # not a comment-only compatibility marker. The queue validates the+ # complete selection and binds one package to pkgs for each lease.+ ordinary_command='go test -race -timeout 50m $pkgs'+ printf '%s\n' "$pkgs" | python3 scripts/release_control/run_go_rest_shard.py \+ --ordinary-command "$ordinary_command" \+ --index "$index" --output "$RUNNER_TEMP/backend-slices"- name: Retain source-listed backend coverageif: always() && needs.changes.outputs.code == 'true'diff --git a/.gitignore b/.gitignoreindex 8edd432d8b..90201eb37f 100644--- a/.gitignore+++ b/.gitignore@@ -274,6 +274,7 @@ scripts/release_control/*!scripts/release_control/resolve_release_promotion_test.py!scripts/release_control/run_go_package_slice.py!scripts/release_control/run_go_package_slice_test.py+!scripts/release_control/run_go_rest_shard.py!scripts/release_control/repo_file_io.py!scripts/release_control/repo_file_io_test.py!scripts/release_control/record_rc_to_ga_rehearsal.pydiff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 4fd8c42bf0..22cefdc2e6 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -1324,12 +1324,19 @@ each worker the ordinary source-listed packages exclude exactly `internal/api`,`internal/monitoring` and `scripts/installtests`. API coverage remains in itsunchanged five shards. Each worker runs the other two slow packages as contiguoussource-listed halves through `scripts/release_control/run_go_package_slice.py`.-The ordinary Go command and both sliced commands run concurrently; every child-must finish successfully before that same protected check can pass. A failed-listing, incomplete JSON execution or failed child cannot be hidden by another-passing command. `GOMAXPROCS=2` bounds the ordinary command to two package-workers, alongside the two sliced binaries: four package workers, not extra-hosted jobs. Documentation-only changes retain the original explicit+`run_go_rest_shard.py` schedules both slices first and ordinary packages in the+same measured order on one four-slot queue. A completed slice releases its slot+to the ordinary package tail, rather than reserving two idle slots. Each ordinary+command covers exactly one selected package with `-race`, unchanged vet and+50-minute timeout, and `-p=1`; `GOMAXPROCS=2` still bounds each binary. Every+admitted child must finish successfully before that same protected check can+pass, even after another child fails. Failed or partial discovery admits no+tasks. Source inventory and package selection must be complete, unique and local+to this module. Incomplete JSON execution or failed children cannot be hidden+by another passing command. The existing artifacts retain every task's exit and+duration alongside the unchanged slice coverage. These are four package workers,+not extra hosted jobs; they do not establish a hosted latency improvement.+Documentation-only changes retain the original explicitruntime-inapplicable results and required check names.The slice runner preserves declaration order, `-race`, uncached execution, thediff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 94ac3f95e1..3a2f593f69 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -4770,6 +4770,7 @@"scripts/release_control/render_release_body.py","scripts/release_control/resolve_release_promotion.py","scripts/release_control/run_go_package_slice.py",+ "scripts/release_control/run_go_rest_shard.py","scripts/release_control/secure_runtime_attestation_v6.py","scripts/release_control/secure_runtime_attestation_v7.py","scripts/release_control/secure_runtime_rootful_attestation_v1.py",diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex ce32bf46a6..2e382c931a 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -4174,6 +4174,8 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {`.github/scripts/go-package-test-seconds.txt 2 "$index")`,"set -euo pipefail","go test -race -timeout 50m $pkgs",+ "python3 scripts/release_control/run_go_rest_shard.py",+ `--index "$index" --output "$RUNNER_TEMP/backend-slices"`,} {if !strings.Contains(backend, required) {t.Fatalf("backend rest shards missing %q", required)@@ -4181,18 +4183,29 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {}for _, required := range []string{- "grep -vE '/(internal/monitoring|scripts/installtests)$'","python3 scripts/release_control/run_go_package_slice_test.py",- "python3 scripts/release_control/run_go_package_slice.py",- "for package in internal/monitoring scripts/installtests; do",- "for pid in \"${pids[@]}\"; do",- "if wait \"$pid\"; then :; else status=1; fi","GOMAXPROCS: '2'", "name: backend-slices-${{ matrix.shard }}",} {if !strings.Contains(backend, required) {t.Fatalf("required backend worker lost exhaustive sliced coverage: %q", required)}}+ worker, err := os.ReadFile(repoFile("scripts", "release_control", "run_go_rest_shard.py"))+ if err != nil {+ t.Fatal(err)+ }+ for _, required := range []string{+ `["go", "list", "./..."]`, `"internal/api"`,+ `SLICED = ("internal/monitoring", "scripts/installtests")`,+ `ORDINARY_COMMAND = "go test -race -timeout 50m $pkgs"`,+ `env["GOFLAGS"] = env.get("GOFLAGS", "") + " -p=1"`,+ `SLOTS = 4`, `ThreadPoolExecutor(max_workers=SLOTS)`,+ `all(result["exit"] == 0 for result in results)`,+ } {+ if !strings.Contains(string(worker), required) {+ t.Fatalf("backend worker lost bounded exhaustive execution: %q", required)+ }+ }shards := workflowJobBlock(t, workflow, "backend-api")indexes := regexp.MustCompile(`(?m)^ index: \[([0-9, ]+)\]$`).FindStringSubmatch(shards)diff --git a/scripts/release_control/run_go_package_slice_test.py b/scripts/release_control/run_go_package_slice_test.pyindex d192e2b6f3..2200d00c4a 100644--- a/scripts/release_control/run_go_package_slice_test.py+++ b/scripts/release_control/run_go_package_slice_test.py@@ -13,6 +13,7 @@ import unittestROOT = Path(__file__).resolve().parents[2]RUNNER = ROOT / "scripts/release_control/run_go_package_slice.py"+SHARD = ROOT / "scripts/release_control/run_go_rest_shard.py"WORKFLOW = ROOT / ".github/workflows/build-and-test.yml"spec = importlib.util.spec_from_file_location("package_slices", RUNNER)runner = importlib.util.module_from_spec(spec)@@ -67,8 +68,21 @@ args = sys.argv[1:]with Path(os.environ['CALLS']).open('a') as f: f.write(json.dumps(args)+'\\n')if args[0] == 'list':print('\\n'.join('github.com/rcourtman/pulse-go-rewrite/'+n for n in- ['internal/api','internal/monitoring','scripts/installtests','internal/a','internal/b','internal/c','internal/d']))- sys.exit(0)+ ['internal/api','internal/monitoring','scripts/installtests']++ ['internal/'+str(i) for i in range(12)]))+ sys.exit(int(os.environ.get('INVENTORY_EXIT','0')))+events = os.environ.get('EVENTS')+if events and '-list' not in args:+ import fcntl, time+ def event(kind):+ with open(events,'a') as f:+ fcntl.flock(f,fcntl.LOCK_EX)+ f.write(json.dumps({'kind':kind,'pid':os.getpid(),'ordinary':'-json' not in args,+ 'goflags':os.environ.get('GOFLAGS',''),+ 'at':time.monotonic()})+'\\n')+ import atexit+ event('start'); atexit.register(event,'end')+ time.sleep(0.06 if '-json' in args else 0.4)if '-list' not in args and '-json' not in args:print('ok ordinary packages 0.01s')sys.exit(int(os.environ.get('ORDINARY_EXIT','0')))@@ -148,13 +162,11 @@ class WorkflowTest(unittest.TestCase):for expected in ('name: Backend tests (${{ matrix.shard }})', 'shard: [rest-0, rest-1]','needs: [changes, test-compile]', "GOMAXPROCS: '2'",- "grep -v '/internal/api$'", "grep -vE '/(internal/monitoring|scripts/installtests)$'",- 'go test -race -timeout 50m $pkgs &',- 'for package in internal/monitoring scripts/installtests; do',- 'scripts/release_control/run_go_package_slice.py',- '--package "$package" --index "$index"',- 'for pid in "${pids[@]}"; do', 'if wait "$pid"; then :; else status=1; fi',- 'exit "$status"', 'name: backend-slices-${{ matrix.shard }}',+ 'python3 scripts/release_control/run_go_rest_shard.py',+ 'ordinary_command=\'go test -race -timeout 50m $pkgs\'',+ '--ordinary-command "$ordinary_command"',+ '--index "$index" --output "$RUNNER_TEMP/backend-slices"',+ 'name: backend-slices-${{ matrix.shard }}',):self.assertIn(expected, block)self.assertNotIn('continue-on-error', block)@@ -167,6 +179,7 @@ class WorkflowTest(unittest.TestCase):self.addCleanup(fixture.doCleanups)root = fixture.rootfor name in ('scripts/release_control/run_go_package_slice.py',+ 'scripts/release_control/run_go_rest_shard.py','.github/scripts/record-internal-api-test-seconds.py','.github/scripts/select-go-package-shard.py','.github/scripts/go-package-test-seconds.txt'):@@ -182,7 +195,8 @@ class WorkflowTest(unittest.TestCase):command = '\n'.join(line[10:] for line in body.splitlines()if line.startswith(' ') or not line.strip())for index, options in enumerate(({}, {'ORDINARY_EXIT':'9'}, {'MODE':'failed-root'},- {'LISTING_EXIT':'7'}, {'CONTROL_EXIT':'5'})):+ {'LISTING_EXIT':'7'}, {'CONTROL_EXIT':'5'},+ {'INVENTORY_EXIT':'8'})):with self.subTest(options=options):calls = root / 'calls'calls.write_text('')@@ -194,15 +208,79 @@ class WorkflowTest(unittest.TestCase):observed = [json.loads(line) for line in calls.read_text().splitlines()]if 'CONTROL_EXIT' in options:self.assertEqual(observed, [])+ elif 'INVENTORY_EXIT' in options:+ # Partial output from a failed discovery admits no task.+ self.assertEqual(observed, [['list', './...']])else:self.assertEqual(sum('-list' in call for call in observed), 2)self.assertEqual(sum('-json' not in call and '-list' not in call- and call[0]=='test' for call in observed), 1)+ and call[0]=='test' for call in observed), 6)+ receipt = json.loads((root / f'output{index}/backend-slices/ordinary-packages.json').read_text())+ self.assertEqual(receipt['slots'], 4)+ self.assertEqual(len(receipt['tasks']), 8)+ self.assertEqual(len({task['package'] for task in receipt['tasks']}), 8)+ self.assertEqual(all(task['exit']==0 for task in receipt['tasks']), not options)++ def test_real_process_queue_reuses_finished_slice_slots_and_drains_all_tasks(self):+ fixture = ExecutedRunnerTest()+ fixture.setUp()+ self.addCleanup(fixture.doCleanups)+ events = fixture.root / 'events'+ output = fixture.root / 'output'+ env = {**os.environ, 'PATH': str(fixture.root)+':'+os.environ['PATH'],+ 'CALLS':str(fixture.root / 'calls'), 'EVENTS':str(events), 'GOMAXPROCS':'2'}+ all_ordinary = []+ for index in (0, 1):+ names = sorted(runner.MODULE+'internal/'+str(i) for i in range(12))+ result = subprocess.run(['python3', str(SHARD), '--index', str(index),+ '--ordinary-command','go test -race -timeout 50m $pkgs',+ '--output',str(output / str(index))],+ input='\n'.join(names[index::2])+'\n',+ cwd=ROOT, env=env, capture_output=True, text=True, timeout=20)+ self.assertEqual(result.returncode, 0, result.stdout+result.stderr)+ receipt = json.loads((output / str(index) / 'ordinary-packages.json').read_text())+ all_ordinary += [task['package'] for task in receipt['tasks'][2:]]+ self.assertEqual(sorted(all_ordinary), sorted(runner.MODULE+'internal/'+str(i) for i in range(12)))+ calls = [json.loads(line) for line in (fixture.root / 'calls').read_text().splitlines()]+ for call in calls:+ if call[0]=='test' and '-list' not in call and '-json' not in call:+ self.assertEqual(call[:-1], ['test','-race','-timeout','50m'])+ active = {}; peak = ordinary_peak = 0+ for event in map(json.loads,events.read_text().splitlines()):+ if event['ordinary']: self.assertEqual(event['goflags'].split()[-1],'-p=1')+ if event['kind']=='start': active[event['pid']] = event['ordinary']+ else: del active[event['pid']]+ peak = max(peak,len(active)); ordinary_peak = max(ordinary_peak,sum(active.values()))+ self.assertEqual(active,{})+ self.assertLessEqual(peak,4)+ self.assertGreaterEqual(ordinary_peak,3, 'finished slices did not release their slots')++ for supplied in ('', '\n'.join(names), '\n'.join(reversed(names[::2]))):+ result = subprocess.run(['python3', str(SHARD), '--index','0',+ '--ordinary-command','go test -race -timeout 50m $pkgs',+ '--output',str(output / 'rejected')],+ input=supplied, cwd=ROOT, env=env,+ capture_output=True, text=True, timeout=20)+ self.assertNotEqual(result.returncode,0)+ self.assertFalse((output / 'rejected').exists())++ def test_inventory_and_launch_failures_cannot_admit_partial_success(self):+ spec = importlib.util.spec_from_file_location('rest_shard', SHARD)+ shard = importlib.util.module_from_spec(spec); spec.loader.exec_module(shard)+ from unittest.mock import patch+ prefix = runner.MODULE+ valid = [prefix+n for n in ('internal/api','internal/monitoring','scripts/installtests','internal/a','internal/b')]+ for inventory in (valid[:-3], valid+valid[:1], valid+['example.invalid/foreign']):+ with patch.object(shard.subprocess, 'run', return_value=subprocess.CompletedProcess([],0,'\n'.join(inventory),'')):+ with self.assertRaises(ValueError): shard.ordinary_packages(0)+ with patch.object(shard.subprocess,'run',side_effect=OSError('start failed')):+ self.assertEqual(shard.run_task(('internal/a',['not-a-command']))['exit'],127)def test_helpers_are_part_of_the_exact_main_controls_copied_to_patches(self):stable_patch = (ROOT / 'scripts/release_control/stable_patch.py').read_text()self.assertIn('"scripts/release_control"', stable_patch)self.assertTrue(RUNNER.is_relative_to(ROOT / 'scripts/release_control'))+ self.assertTrue(SHARD.is_relative_to(ROOT / 'scripts/release_control'))self.assertTrue(Path(__file__).is_relative_to(ROOT / 'scripts/release_control'))self.assertIn('python3 scripts/release_control/run_go_package_slice_test.py', job('backend'))diff --git a/scripts/release_control/run_go_rest_shard.py b/scripts/release_control/run_go_rest_shard.pynew file mode 100644index 0000000000..adc738baaa--- /dev/null+++ b/scripts/release_control/run_go_rest_shard.py@@ -0,0 +1,113 @@+#!/usr/bin/env python3+"""Share four package slots between source-listed slices and ordinary packages.++The old recipe reserved two ordinary package workers even after both slow+slices finished. A single queue releases those slots without adding jobs or+increasing the four-binary envelope. Timing weights schedule, never select,+the source-derived ordinary package inventory.+"""++from __future__ import annotations++import argparse+from concurrent.futures import ThreadPoolExecutor+import json+import os+from pathlib import Path+import subprocess+import sys+import time+++ROOT = Path(__file__).resolve().parents[2]+MODULE = "github.com/rcourtman/pulse-go-rewrite/"+SLICED = ("internal/monitoring", "scripts/installtests")+SLOTS = 4+ORDINARY_COMMAND = "go test -race -timeout 50m $pkgs"+++def ordinary_packages(index: int) -> list[str]:+ listing = subprocess.run(["go", "list", "./..."], cwd=ROOT,+ capture_output=True, text=True, check=True)+ packages = listing.stdout.splitlines()+ required = {MODULE + name for name in (*SLICED, "internal/api")}+ if (not required.issubset(packages) or len(packages) != len(set(packages))+ or any(not name.startswith(MODULE) for name in packages)):+ raise ValueError("incomplete, duplicate or foreign source package inventory")+ ordinary = [name for name in packages if name not in required]+ selection = subprocess.run(+ [sys.executable, str(ROOT / ".github/scripts/select-go-package-shard.py"),+ str(ROOT / ".github/scripts/go-package-test-seconds.txt"), "2", str(index)],+ cwd=ROOT, input="\n".join(ordinary) + "\n",+ capture_output=True, text=True, check=True)+ selected = selection.stdout.splitlines()+ if (not selected or len(selected) != len(set(selected))+ or not set(selected).issubset(ordinary)):+ raise ValueError("empty or invalid ordinary package shard")+ return selected+++def tasks(packages: list[str], index: int, output: Path,+ ordinary_command: str) -> list[tuple[str, list[str]]]:+ sliced = [(name, [sys.executable, str(ROOT / "scripts/release_control/run_go_package_slice.py"),+ "--package", name, "--index", str(index),+ "--output", str(output / name.rsplit("/", 1)[-1])])+ for name in SLICED]+ # One package per lease. -p=1 bounds its compiler as well as its binary;+ # GOMAXPROCS=2 still bounds each binary, exactly as in the previous recipe.+ # A completed slice can immediately admit another ordinary package.+ return sliced + [(name, ["bash", "-euo", "pipefail", "-c", ordinary_command])+ for name in packages]+++def run_task(task: tuple[str, list[str]]) -> dict:+ name, command = task+ start = time.monotonic()+ try:+ env = dict(os.environ)+ if name not in SLICED:+ env["pkgs"] = name+ env["GOFLAGS"] = env.get("GOFLAGS", "") + " -p=1"+ result = subprocess.run(command, cwd=ROOT, env=env)+ code = result.returncode+ except OSError as error:+ print(f"Could not start backend package {name}: {error}", file=sys.stderr)+ code = 127+ return {"package": name, "exit": code,+ "seconds": round(time.monotonic() - start, 3)}+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--index", type=int, choices=(0, 1), required=True)+ parser.add_argument("--output", type=Path, required=True)+ # Only the existing fixed Go recipe is supported, not arbitrary shell+ # input. Keeping it in the workflow also preserves stable-shipped source+ # contract tests when main's controls are copied to a mechanical patch.+ parser.add_argument("--ordinary-command", choices=(ORDINARY_COMMAND,), required=True)+ args = parser.parse_args()+ try:+ packages = ordinary_packages(args.index)+ if sys.stdin.read().splitlines() != packages:+ raise ValueError("supplied package order differs from the complete source selection")+ except (OSError, ValueError, subprocess.CalledProcessError) as error:+ print(f"Backend package admission failed: {error}", file=sys.stderr)+ if isinstance(error, subprocess.CalledProcessError):+ sys.stdout.write(error.stdout or "")+ sys.stderr.write(error.stderr or "")+ return 1+ args.output.mkdir(parents=True, exist_ok=True)+ print("Ordinary packages in this shard:\n" + "\n".join(packages), flush=True)+ start = time.monotonic()+ # Drain every admitted task even when a sibling fails. Executor.map retains+ # source scheduling order in the receipt, not concurrent completion order.+ with ThreadPoolExecutor(max_workers=SLOTS) as pool:+ results = list(pool.map(run_task, tasks(packages, args.index, args.output, args.ordinary_command)))+ receipt = {"index": args.index, "slots": SLOTS, "tasks": results,+ "seconds": round(time.monotonic() - start, 3)}+ (args.output / "ordinary-packages.json").write_text(json.dumps(receipt, indent=2) + "\n")+ return 0 if all(result["exit"] == 0 for result in results) else 1+++if __name__ == "__main__":+ raise SystemExit(main())