Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Balance the existing non-API race-test checks by measured package cost

pulse-triage[bot]7be63facb9a9
diff --git a/.github/scripts/go-package-test-seconds.txt b/.github/scripts/go-package-test-seconds.txt
new file mode 100644
index 0000000000..d28c71b291
--- /dev/null
+++ b/.github/scripts/go-package-test-seconds.txt
@@ -0,0 +1,35 @@
+# Non-API Go package race-test seconds on GitHub ubuntu-24.04.
+# Median of successful rest shards in Build and Test runs 37451091575
+# (b5444cf8) and 37451828694 (bd0ef398), collected 2026-10-06.
+# Package elapsed times overlap because go test runs packages concurrently;
+# these are balancing weights, not predicted wall-clock durations.
+# Weights move whole packages between the two existing required checks.
+# The checked-out go list supplies coverage, including new/unmeasured packages.
+# Stale entries cannot add, omit or duplicate a package. DEFAULT_WEIGHT is
+# the mean of measured test packages below 20 seconds in those same runs.
+# Refresh from complete passing rest-shard logs when imbalance returns.
+DEFAULT_WEIGHT 2.681
+github.com/rcourtman/pulse-go-rewrite/cmd/hashpw 22.616
+github.com/rcourtman/pulse-go-rewrite/internal/agentupdate 35.681
+github.com/rcourtman/pulse-go-rewrite/internal/ai 30.231
+github.com/rcourtman/pulse-go-rewrite/internal/ai/chat 20.800
+github.com/rcourtman/pulse-go-rewrite/internal/ai/tools 63.876
+github.com/rcourtman/pulse-go-rewrite/internal/alerts 27.456
+github.com/rcourtman/pulse-go-rewrite/internal/api/configapi 34.133
+github.com/rcourtman/pulse-go-rewrite/internal/config 228.780
+github.com/rcourtman/pulse-go-rewrite/internal/dockeragent 22.758
+github.com/rcourtman/pulse-go-rewrite/internal/mock 129.980
+github.com/rcourtman/pulse-go-rewrite/internal/monitoring 201.733
+github.com/rcourtman/pulse-go-rewrite/internal/notifications 121.213
+github.com/rcourtman/pulse-go-rewrite/internal/recovery/store 23.478
+github.com/rcourtman/pulse-go-rewrite/internal/relay 70.288
+github.com/rcourtman/pulse-go-rewrite/internal/telemetry 38.002
+github.com/rcourtman/pulse-go-rewrite/internal/truenas 48.556
+github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources 124.084
+github.com/rcourtman/pulse-go-rewrite/internal/updates 26.972
+github.com/rcourtman/pulse-go-rewrite/internal/websocket 27.300
+github.com/rcourtman/pulse-go-rewrite/pkg/auth 26.903
+github.com/rcourtman/pulse-go-rewrite/pkg/metrics 324.930
+github.com/rcourtman/pulse-go-rewrite/pkg/server 26.114
+github.com/rcourtman/pulse-go-rewrite/scripts/installtests 206.606
+github.com/rcourtman/pulse-go-rewrite/tests/migration 48.435
diff --git a/.github/scripts/select-go-package-shard.py b/.github/scripts/select-go-package-shard.py
new file mode 100644
index 0000000000..ab03848745
--- /dev/null
+++ b/.github/scripts/select-go-package-shard.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Balance whole Go packages without changing which tests run.
+
+Usage: go list ... | select-go-package-shard.py WEIGHTS COUNT INDEX
+
+The checked-out source supplies the package universe, never the weights file.
+Assign longest measured packages first to the lightest shard, breaking ties by
+package name and shard index. Print selected packages in their original order.
+Unlike internal/api's test slices, separate package binaries have no shared
+test order to preserve. Missing/renamed weights affect balance, not coverage.
+"""
+
+from __future__ import annotations
+
+import argparse
+from decimal import Decimal, InvalidOperation
+from pathlib import Path
+import re
+import sys
+
+
+PACKAGE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._~+/-]*\Z")
+
+
+def read_weights(path: Path) -> tuple[dict[str, Decimal], Decimal]:
+ weights: dict[str, Decimal] = {}
+ default = Decimal("1")
+ seen: set[str] = set()
+ for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ if not line.strip() or line.lstrip().startswith("#"):
+ continue
+ fields = line.split()
+ if len(fields) != 2:
+ raise ValueError(f"invalid weights line {number}: expected name and seconds")
+ name, raw = fields
+ if name != "DEFAULT_WEIGHT" and not PACKAGE.fullmatch(name):
+ raise ValueError(f"invalid package name on weights line {number}")
+ if name in seen:
+ raise ValueError(f"duplicate weight on line {number}: {name}")
+ seen.add(name)
+ try:
+ seconds = Decimal(raw)
+ except InvalidOperation:
+ raise ValueError(f"invalid seconds on weights line {number}") from None
+ if not seconds.is_finite() or not 0 < seconds <= 86400:
+ raise ValueError(f"seconds must be finite and within (0, 86400] on weights line {number}")
+ if name == "DEFAULT_WEIGHT":
+ default = seconds
+ else:
+ weights[name] = seconds
+ return weights, default
+
+
+def assign(packages: list[str], weights: dict[str, Decimal], default: Decimal, count: int) -> dict[str, int]:
+ if count < 1 or len(packages) < count:
+ raise ValueError(f"{len(packages)} packages cannot fill {count} shards")
+ if len(set(packages)) != len(packages):
+ raise ValueError("duplicate package in source list")
+ if any(not PACKAGE.fullmatch(name) for name in packages):
+ raise ValueError("invalid package in source list")
+ loads = [Decimal(0)] * count
+ assignment: dict[str, int] = {}
+ for name in sorted(packages, key=lambda name: (-weights.get(name, default), name)):
+ shard = min(range(count), key=lambda index: (loads[index], index))
+ assignment[name] = shard
+ loads[shard] += weights.get(name, default)
+ return assignment
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("weights", type=Path)
+ parser.add_argument("count", type=int)
+ parser.add_argument("index", type=int)
+ args = parser.parse_args()
+ try:
+ if args.count < 1 or not 0 <= args.index < args.count:
+ raise ValueError("shard index must be within the positive shard count")
+ weights, default = read_weights(args.weights)
+ packages = [line for line in sys.stdin.read().splitlines() if line]
+ assignment = assign(packages, weights, default, args.count)
+ except (OSError, ValueError, InvalidOperation) as error:
+ print(f"package shard selection failed: {error}", file=sys.stderr)
+ return 2
+ for name in packages:
+ if assignment[name] == args.index:
+ print(name)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 4fd206ae05..fa545bfde3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -273,8 +273,8 @@ jobs:
strategy:
fail-fast: false
matrix:
- # internal/api has its own sharded job below; every other package
- # splits across these two.
+ # internal/api has its own sharded job below. Measured weights balance
+ # every other whole package across these two unchanged required checks.
shard: [rest-0, rest-1]
steps:
@@ -310,15 +310,20 @@ jobs:
if: needs.changes.outputs.code == 'true'
env:
PULSE_DATA_DIR: /tmp/pulse-test-data
- # Packages other than internal/api are split deterministically by
- # list position.
+ # Source, not the timing file, defines coverage. Longest measured
+ # packages go to the lightest shard; new packages use a default weight.
+ # Package binaries are independent, so only internal/api needs the
+ # contiguous, order-preserving test slices used in its separate job.
run: |
set -euo pipefail
case "${{ matrix.shard }}" in
- rest-0) pkgs=$(go list ./... | grep -v '/internal/api$' | awk 'NR % 2 == 0') ;;
- rest-1) pkgs=$(go list ./... | grep -v '/internal/api$' | awk 'NR % 2 == 1') ;;
+ rest-0) index=0 ;;
+ rest-1) index=1 ;;
*) echo "unknown shard ${{ matrix.shard }}" >&2; exit 1 ;;
esac
+ pkgs=$(go list ./... | grep -v '/internal/api$' \
+ | python3 .github/scripts/select-go-package-shard.py \
+ .github/scripts/go-package-test-seconds.txt 2 "$index")
if [ -z "$pkgs" ]; then
echo "shard resolved to an empty package list" >&2
exit 1
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 86549dfa8a..9e120752f7 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -3923,6 +3923,9 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
"name: Backend tests (${{ matrix.shard }})",
"shard: [rest-0, rest-1]",
"grep -v '/internal/api$'",
+ "python3 .github/scripts/select-go-package-shard.py",
+ `.github/scripts/go-package-test-seconds.txt 2 "$index")`,
+ "set -euo pipefail",
"go test -race -timeout 50m $pkgs",
} {
if !strings.Contains(backend, required) {
diff --git a/scripts/tests/test_go_package_shards.py b/scripts/tests/test_go_package_shards.py
new file mode 100644
index 0000000000..7b05a13aef
--- /dev/null
+++ b/scripts/tests/test_go_package_shards.py
@@ -0,0 +1,100 @@
+"""Whole-package balancing must never trade exhaustive CI for faster checks."""
+
+import importlib.util
+from pathlib import Path
+import random
+import subprocess
+import tempfile
+import unittest
+from decimal import Decimal
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SELECTOR = ROOT / ".github/scripts/select-go-package-shard.py"
+spec = importlib.util.spec_from_file_location("package_shards", SELECTOR)
+selector = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(selector)
+
+
+class PackageShardsTest(unittest.TestCase):
+ def cli(self, text, weights="DEFAULT_WEIGHT 1\n", count=2, index=0):
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "weights.txt"
+ path.write_text(weights, encoding="utf-8")
+ return subprocess.run(
+ ["python3", str(SELECTOR), str(path), str(count), str(index)],
+ input=text, text=True, capture_output=True,
+ )
+
+ def test_measured_heavy_packages_are_separated(self):
+ # The previous odd/even split put both slow packages on shard 1.
+ packages = ["example/a", "example/b", "example/c", "example/d"]
+ weights = {"example/a": Decimal(100), "example/c": Decimal(80)}
+ assigned = selector.assign(packages, weights, Decimal(1), 2)
+ self.assertNotEqual(assigned["example/a"], assigned["example/c"])
+ costs = [sum(weights.get(p, Decimal(1)) for p in packages if assigned[p] == i) for i in range(2)]
+ self.assertEqual(costs, [Decimal(100), Decimal(82)])
+ self.assertEqual(sum(costs), Decimal(182))
+
+ def test_exhaustive_nonempty_deterministic_assignment_under_churn(self):
+ randomizer = random.Random(60106)
+ for count in range(1, 9):
+ for size in (count, count + 1, count * 17):
+ with self.subTest(count=count, size=size):
+ packages = [f"example/p{i}" for i in range(size)]
+ randomizer.shuffle(packages)
+ weights = {p: Decimal(randomizer.randint(1, 500)) for p in packages[::3]}
+ weights["example/deleted"] = Decimal(9999)
+ assigned = selector.assign(packages, weights, Decimal(1), count)
+ self.assertEqual(set(assigned), set(packages))
+ self.assertEqual(set(assigned.values()), set(range(count)))
+ self.assertEqual(assigned, selector.assign(list(reversed(packages)), weights, Decimal(1), count))
+
+ def test_cli_keeps_source_order_and_new_unmeasured_packages(self):
+ packages = ["example/new", "example/z", "example/a", "example/b", "example/renamed"]
+ weights = "# stale names are not coverage\nDEFAULT_WEIGHT 2.5\nexample/deleted 100\nexample/z 20\n"
+ selected = []
+ for index in range(2):
+ result = self.cli("\n".join(packages) + "\n", weights, index=index)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ names = result.stdout.splitlines()
+ self.assertTrue(names)
+ self.assertEqual(names, [p for p in packages if p in names])
+ selected.extend(names)
+ self.assertCountEqual(selected, packages)
+
+ def test_equal_weights_and_decimal_ties_are_stable(self):
+ packages = ["example/d", "example/c", "example/b", "example/a"]
+ assigned = selector.assign(packages, {}, Decimal("0.1"), 2)
+ self.assertEqual(assigned, {"example/a": 0, "example/b": 1, "example/c": 0, "example/d": 1})
+
+ def test_invalid_weights_fail_before_printing_any_package(self):
+ for weights in ("example/a", "example/a nope", "example/a NaN", "example/a Infinity", "example/a -1", "example/a 0", "example/a 1e99999999", "example/a 1\nexample/a 2", "DEFAULT_WEIGHT 1\nDEFAULT_WEIGHT 2", "-option 1"):
+ with self.subTest(weights=weights):
+ result = self.cli("example/a\nexample/b\n", weights)
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(result.stdout, "")
+ self.assertIn("package shard selection failed:", result.stderr)
+
+ def test_invalid_source_or_shard_fail_closed(self):
+ for text, count, index in (("", 2, 0), ("example/a\n", 2, 0), ("example/a\nexample/a\n", 2, 0), ("-option\nexample/a\n", 2, 0), ("example/a extra\nexample/b\n", 2, 0), ("example/a\nexample/b\n", 0, 0), ("example/a\nexample/b\n", 2, -1), ("example/a\nexample/b\n", 2, 2)):
+ with self.subTest(text=text, count=count, index=index):
+ result = self.cli(text, count=count, index=index)
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(result.stdout, "")
+
+ def test_missing_weights_cannot_silently_fall_back(self):
+ with tempfile.TemporaryDirectory() as directory:
+ result = subprocess.run(["python3", str(SELECTOR), str(Path(directory) / "absent"), "2", "0"], input="example/a\nexample/b\n", text=True, capture_output=True)
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(result.stdout, "")
+
+ def test_committed_weights_are_valid_and_positive(self):
+ weights, default = selector.read_weights(ROOT / ".github/scripts/go-package-test-seconds.txt")
+ self.assertGreater(default, 0)
+ self.assertGreater(len(weights), 0)
+ self.assertTrue(all(value > 0 for value in weights.values()))
+
+
+if __name__ == "__main__":
+ unittest.main()