Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Balance the existing non-API race-test checks by measured package cost
pulse-triage[bot]7be63facb9a9
diff --git a/.github/scripts/go-package-test-seconds.txt b/.github/scripts/go-package-test-seconds.txtnew file mode 100644index 0000000000..d28c71b291--- /dev/null+++ b/.github/scripts/go-package-test-seconds.txt@@ -0,0 +1,35 @@+# Non-API Go package race-test seconds on GitHub ubuntu-24.04.+# Median of successful rest shards in Build and Test runs 37451091575+# (b5444cf8) and 37451828694 (bd0ef398), collected 2026-10-06.+# Package elapsed times overlap because go test runs packages concurrently;+# these are balancing weights, not predicted wall-clock durations.+# Weights move whole packages between the two existing required checks.+# The checked-out go list supplies coverage, including new/unmeasured packages.+# Stale entries cannot add, omit or duplicate a package. DEFAULT_WEIGHT is+# the mean of measured test packages below 20 seconds in those same runs.+# Refresh from complete passing rest-shard logs when imbalance returns.+DEFAULT_WEIGHT 2.681+github.com/rcourtman/pulse-go-rewrite/cmd/hashpw 22.616+github.com/rcourtman/pulse-go-rewrite/internal/agentupdate 35.681+github.com/rcourtman/pulse-go-rewrite/internal/ai 30.231+github.com/rcourtman/pulse-go-rewrite/internal/ai/chat 20.800+github.com/rcourtman/pulse-go-rewrite/internal/ai/tools 63.876+github.com/rcourtman/pulse-go-rewrite/internal/alerts 27.456+github.com/rcourtman/pulse-go-rewrite/internal/api/configapi 34.133+github.com/rcourtman/pulse-go-rewrite/internal/config 228.780+github.com/rcourtman/pulse-go-rewrite/internal/dockeragent 22.758+github.com/rcourtman/pulse-go-rewrite/internal/mock 129.980+github.com/rcourtman/pulse-go-rewrite/internal/monitoring 201.733+github.com/rcourtman/pulse-go-rewrite/internal/notifications 121.213+github.com/rcourtman/pulse-go-rewrite/internal/recovery/store 23.478+github.com/rcourtman/pulse-go-rewrite/internal/relay 70.288+github.com/rcourtman/pulse-go-rewrite/internal/telemetry 38.002+github.com/rcourtman/pulse-go-rewrite/internal/truenas 48.556+github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources 124.084+github.com/rcourtman/pulse-go-rewrite/internal/updates 26.972+github.com/rcourtman/pulse-go-rewrite/internal/websocket 27.300+github.com/rcourtman/pulse-go-rewrite/pkg/auth 26.903+github.com/rcourtman/pulse-go-rewrite/pkg/metrics 324.930+github.com/rcourtman/pulse-go-rewrite/pkg/server 26.114+github.com/rcourtman/pulse-go-rewrite/scripts/installtests 206.606+github.com/rcourtman/pulse-go-rewrite/tests/migration 48.435diff --git a/.github/scripts/select-go-package-shard.py b/.github/scripts/select-go-package-shard.pynew file mode 100644index 0000000000..ab03848745--- /dev/null+++ b/.github/scripts/select-go-package-shard.py@@ -0,0 +1,92 @@+#!/usr/bin/env python3+"""Balance whole Go packages without changing which tests run.++Usage: go list ... | select-go-package-shard.py WEIGHTS COUNT INDEX++The checked-out source supplies the package universe, never the weights file.+Assign longest measured packages first to the lightest shard, breaking ties by+package name and shard index. Print selected packages in their original order.+Unlike internal/api's test slices, separate package binaries have no shared+test order to preserve. Missing/renamed weights affect balance, not coverage.+"""++from __future__ import annotations++import argparse+from decimal import Decimal, InvalidOperation+from pathlib import Path+import re+import sys+++PACKAGE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._~+/-]*\Z")+++def read_weights(path: Path) -> tuple[dict[str, Decimal], Decimal]:+ weights: dict[str, Decimal] = {}+ default = Decimal("1")+ seen: set[str] = set()+ for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):+ if not line.strip() or line.lstrip().startswith("#"):+ continue+ fields = line.split()+ if len(fields) != 2:+ raise ValueError(f"invalid weights line {number}: expected name and seconds")+ name, raw = fields+ if name != "DEFAULT_WEIGHT" and not PACKAGE.fullmatch(name):+ raise ValueError(f"invalid package name on weights line {number}")+ if name in seen:+ raise ValueError(f"duplicate weight on line {number}: {name}")+ seen.add(name)+ try:+ seconds = Decimal(raw)+ except InvalidOperation:+ raise ValueError(f"invalid seconds on weights line {number}") from None+ if not seconds.is_finite() or not 0 < seconds <= 86400:+ raise ValueError(f"seconds must be finite and within (0, 86400] on weights line {number}")+ if name == "DEFAULT_WEIGHT":+ default = seconds+ else:+ weights[name] = seconds+ return weights, default+++def assign(packages: list[str], weights: dict[str, Decimal], default: Decimal, count: int) -> dict[str, int]:+ if count < 1 or len(packages) < count:+ raise ValueError(f"{len(packages)} packages cannot fill {count} shards")+ if len(set(packages)) != len(packages):+ raise ValueError("duplicate package in source list")+ if any(not PACKAGE.fullmatch(name) for name in packages):+ raise ValueError("invalid package in source list")+ loads = [Decimal(0)] * count+ assignment: dict[str, int] = {}+ for name in sorted(packages, key=lambda name: (-weights.get(name, default), name)):+ shard = min(range(count), key=lambda index: (loads[index], index))+ assignment[name] = shard+ loads[shard] += weights.get(name, default)+ return assignment+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("weights", type=Path)+ parser.add_argument("count", type=int)+ parser.add_argument("index", type=int)+ args = parser.parse_args()+ try:+ if args.count < 1 or not 0 <= args.index < args.count:+ raise ValueError("shard index must be within the positive shard count")+ weights, default = read_weights(args.weights)+ packages = [line for line in sys.stdin.read().splitlines() if line]+ assignment = assign(packages, weights, default, args.count)+ except (OSError, ValueError, InvalidOperation) as error:+ print(f"package shard selection failed: {error}", file=sys.stderr)+ return 2+ for name in packages:+ if assignment[name] == args.index:+ print(name)+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.ymlindex 4fd206ae05..fa545bfde3 100644--- a/.github/workflows/build-and-test.yml+++ b/.github/workflows/build-and-test.yml@@ -273,8 +273,8 @@ jobs:strategy:fail-fast: falsematrix:- # internal/api has its own sharded job below; every other package- # splits across these two.+ # internal/api has its own sharded job below. Measured weights balance+ # every other whole package across these two unchanged required checks.shard: [rest-0, rest-1]steps:@@ -310,15 +310,20 @@ jobs:if: needs.changes.outputs.code == 'true'env:PULSE_DATA_DIR: /tmp/pulse-test-data- # Packages other than internal/api are split deterministically by- # list position.+ # Source, not the timing file, defines coverage. Longest measured+ # packages go to the lightest shard; new packages use a default weight.+ # Package binaries are independent, so only internal/api needs the+ # contiguous, order-preserving test slices used in its separate job.run: |set -euo pipefailcase "${{ matrix.shard }}" in- rest-0) pkgs=$(go list ./... | grep -v '/internal/api$' | awk 'NR % 2 == 0') ;;- rest-1) pkgs=$(go list ./... | grep -v '/internal/api$' | awk 'NR % 2 == 1') ;;+ rest-0) index=0 ;;+ rest-1) index=1 ;;*) echo "unknown shard ${{ matrix.shard }}" >&2; exit 1 ;;esac+ pkgs=$(go list ./... | grep -v '/internal/api$' \+ | python3 .github/scripts/select-go-package-shard.py \+ .github/scripts/go-package-test-seconds.txt 2 "$index")if [ -z "$pkgs" ]; thenecho "shard resolved to an empty package list" >&2exit 1diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex 86549dfa8a..9e120752f7 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -3923,6 +3923,9 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {"name: Backend tests (${{ matrix.shard }})","shard: [rest-0, rest-1]","grep -v '/internal/api$'",+ "python3 .github/scripts/select-go-package-shard.py",+ `.github/scripts/go-package-test-seconds.txt 2 "$index")`,+ "set -euo pipefail","go test -race -timeout 50m $pkgs",} {if !strings.Contains(backend, required) {diff --git a/scripts/tests/test_go_package_shards.py b/scripts/tests/test_go_package_shards.pynew file mode 100644index 0000000000..7b05a13aef--- /dev/null+++ b/scripts/tests/test_go_package_shards.py@@ -0,0 +1,100 @@+"""Whole-package balancing must never trade exhaustive CI for faster checks."""++import importlib.util+from pathlib import Path+import random+import subprocess+import tempfile+import unittest+from decimal import Decimal+++ROOT = Path(__file__).resolve().parents[2]+SELECTOR = ROOT / ".github/scripts/select-go-package-shard.py"+spec = importlib.util.spec_from_file_location("package_shards", SELECTOR)+selector = importlib.util.module_from_spec(spec)+spec.loader.exec_module(selector)+++class PackageShardsTest(unittest.TestCase):+ def cli(self, text, weights="DEFAULT_WEIGHT 1\n", count=2, index=0):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "weights.txt"+ path.write_text(weights, encoding="utf-8")+ return subprocess.run(+ ["python3", str(SELECTOR), str(path), str(count), str(index)],+ input=text, text=True, capture_output=True,+ )++ def test_measured_heavy_packages_are_separated(self):+ # The previous odd/even split put both slow packages on shard 1.+ packages = ["example/a", "example/b", "example/c", "example/d"]+ weights = {"example/a": Decimal(100), "example/c": Decimal(80)}+ assigned = selector.assign(packages, weights, Decimal(1), 2)+ self.assertNotEqual(assigned["example/a"], assigned["example/c"])+ costs = [sum(weights.get(p, Decimal(1)) for p in packages if assigned[p] == i) for i in range(2)]+ self.assertEqual(costs, [Decimal(100), Decimal(82)])+ self.assertEqual(sum(costs), Decimal(182))++ def test_exhaustive_nonempty_deterministic_assignment_under_churn(self):+ randomizer = random.Random(60106)+ for count in range(1, 9):+ for size in (count, count + 1, count * 17):+ with self.subTest(count=count, size=size):+ packages = [f"example/p{i}" for i in range(size)]+ randomizer.shuffle(packages)+ weights = {p: Decimal(randomizer.randint(1, 500)) for p in packages[::3]}+ weights["example/deleted"] = Decimal(9999)+ assigned = selector.assign(packages, weights, Decimal(1), count)+ self.assertEqual(set(assigned), set(packages))+ self.assertEqual(set(assigned.values()), set(range(count)))+ self.assertEqual(assigned, selector.assign(list(reversed(packages)), weights, Decimal(1), count))++ def test_cli_keeps_source_order_and_new_unmeasured_packages(self):+ packages = ["example/new", "example/z", "example/a", "example/b", "example/renamed"]+ weights = "# stale names are not coverage\nDEFAULT_WEIGHT 2.5\nexample/deleted 100\nexample/z 20\n"+ selected = []+ for index in range(2):+ result = self.cli("\n".join(packages) + "\n", weights, index=index)+ self.assertEqual(result.returncode, 0, result.stderr)+ names = result.stdout.splitlines()+ self.assertTrue(names)+ self.assertEqual(names, [p for p in packages if p in names])+ selected.extend(names)+ self.assertCountEqual(selected, packages)++ def test_equal_weights_and_decimal_ties_are_stable(self):+ packages = ["example/d", "example/c", "example/b", "example/a"]+ assigned = selector.assign(packages, {}, Decimal("0.1"), 2)+ self.assertEqual(assigned, {"example/a": 0, "example/b": 1, "example/c": 0, "example/d": 1})++ def test_invalid_weights_fail_before_printing_any_package(self):+ for weights in ("example/a", "example/a nope", "example/a NaN", "example/a Infinity", "example/a -1", "example/a 0", "example/a 1e99999999", "example/a 1\nexample/a 2", "DEFAULT_WEIGHT 1\nDEFAULT_WEIGHT 2", "-option 1"):+ with self.subTest(weights=weights):+ result = self.cli("example/a\nexample/b\n", weights)+ self.assertEqual(result.returncode, 2)+ self.assertEqual(result.stdout, "")+ self.assertIn("package shard selection failed:", result.stderr)++ def test_invalid_source_or_shard_fail_closed(self):+ for text, count, index in (("", 2, 0), ("example/a\n", 2, 0), ("example/a\nexample/a\n", 2, 0), ("-option\nexample/a\n", 2, 0), ("example/a extra\nexample/b\n", 2, 0), ("example/a\nexample/b\n", 0, 0), ("example/a\nexample/b\n", 2, -1), ("example/a\nexample/b\n", 2, 2)):+ with self.subTest(text=text, count=count, index=index):+ result = self.cli(text, count=count, index=index)+ self.assertEqual(result.returncode, 2)+ self.assertEqual(result.stdout, "")++ def test_missing_weights_cannot_silently_fall_back(self):+ with tempfile.TemporaryDirectory() as directory:+ result = subprocess.run(["python3", str(SELECTOR), str(Path(directory) / "absent"), "2", "0"], input="example/a\nexample/b\n", text=True, capture_output=True)+ self.assertEqual(result.returncode, 2)+ self.assertEqual(result.stdout, "")++ def test_committed_weights_are_valid_and_positive(self):+ weights, default = selector.read_weights(ROOT / ".github/scripts/go-package-test-seconds.txt")+ self.assertGreater(default, 0)+ self.assertGreater(len(weights), 0)+ self.assertTrue(all(value > 0 for value in weights.values()))+++if __name__ == "__main__":+ unittest.main()