Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Balance isolated smoke suites with measured scheduling hints
pulse-triage[bot]db76f62798d8
diff --git a/.github/scripts/smoke-test-seconds.txt b/.github/scripts/smoke-test-seconds.txtnew file mode 100644index 0000000000..014e76e242--- /dev/null+++ b/.github/scripts/smoke-test-seconds.txt@@ -0,0 +1,22 @@+# Script smoke suite seconds in ordinary PR CI, 2026-10-26.+# Complete successful run 40313223658 at a5659362bf4b7466d2f308444bdd21849aeceb4c;+# jobs 121540828495 / 121540828581, measured between suite headers.+# These are scheduling hints, never the suite inventory or deadlines.+# The current discovered tree supplies every suite; stale hints cannot add one.+# Suites below 15 seconds use their measured mean, as do new/unmeasured suites.+# Refresh from complete passing shard logs if imbalance returns.+DEFAULT_WEIGHT 1.881+test_admin_docs.py 25.874+test_agent_log_docs.py 16.473+test_api_failure_index.py 15.110+test_authentication_diagnostic_docs.py 21.495+test_auto_update_service_state.py 28.879+test_docker_troubleshooting_docs.py 15.349+test_memory_troubleshooting_docs.py 43.941+test_notification_troubleshooting_docs.py 24.617+test_npm_audit_retry.py 30.170+test_performance_troubleshooting_docs.py 248.554+test_server_installer_stop.py 38.558+test_troubleshooting_logs.py 59.703+test_truenas_docs.py 27.603+test_vm_disk_diagnostics.py 80.059diff --git a/scripts/select-smoke-test-shard.py b/scripts/select-smoke-test-shard.pynew file mode 100644index 0000000000..ebda6869f7--- /dev/null+++ b/scripts/select-smoke-test-shard.py@@ -0,0 +1,87 @@+#!/usr/bin/env python3+"""Balance whole smoke suites; discovery, never timing hints, defines coverage."""+from __future__ import annotations++import argparse+from decimal import Decimal, InvalidOperation+from pathlib import Path+import sys+++def suite_name(name: str) -> bool:+ return Path(name).name == name and (+ name.startswith("test_") and name.endswith(".py")+ or name.startswith("test-") and name.endswith(".sh")+ )+++def read_weights(path: Path) -> tuple[Decimal, dict[str, Decimal]]:+ default = None+ weights: dict[str, Decimal] = {}+ for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):+ line = raw.split("#", 1)[0].strip()+ if not line:+ continue+ fields = line.split()+ if len(fields) != 2:+ raise ValueError(f"invalid timing hint at line {number}")+ name, value = fields+ try:+ seconds = Decimal(value)+ except InvalidOperation as error:+ raise ValueError(f"invalid timing value at line {number}") from error+ if not seconds.is_finite() or seconds <= 0:+ raise ValueError(f"timing hint must be finite and positive at line {number}")+ if name == "DEFAULT_WEIGHT":+ if default is not None:+ raise ValueError("duplicate DEFAULT_WEIGHT")+ default = seconds+ else:+ if not suite_name(name) or name in weights:+ raise ValueError(f"invalid or duplicate suite timing at line {number}")+ weights[name] = seconds+ if default is None:+ raise ValueError("missing DEFAULT_WEIGHT")+ return default, weights+++def select(paths: list[str], default: Decimal, weights: dict[str, Decimal],+ count: int, index: int) -> list[str]:+ if not 1 <= count <= 64 or not 0 <= index < count:+ raise ValueError("invalid smoke shard count or index")+ if len(paths) != len(set(paths)) or any(+ not path or "\0" in path or not suite_name(Path(path).name) for path in paths+ ):+ raise ValueError("invalid or duplicate discovered smoke suite")+ loads = [Decimal(0) for _ in range(count)]+ shards: list[list[str]] = [[] for _ in range(count)]+ # Stable ties keep unmeasured suites balanced too. We never iterate hints+ # to construct the inventory, so new, renamed and removed suites are safe.+ for path in sorted(paths, key=lambda item: (-weights.get(Path(item).name, default), item)):+ shard = min(range(count), key=lambda candidate: (loads[candidate], candidate))+ shards[shard].append(path)+ loads[shard] += weights.get(Path(path).name, default)+ # Retain the runner's sorted execution within each isolated shard. No+ # suite's command, deadline, process cleanup or failure handling changes.+ return sorted(shards[index])+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("weights", type=Path)+ parser.add_argument("count", type=int)+ parser.add_argument("index", type=int)+ args = parser.parse_args()+ try:+ default, weights = read_weights(args.weights)+ selected = select(sys.stdin.read().splitlines(), default, weights, args.count, args.index)+ except (OSError, UnicodeError, ValueError) as error:+ print(f"Cannot partition complete smoke inventory: {error}", file=sys.stderr)+ return 1+ if selected:+ print("\n".join(selected))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/tests/run.sh b/scripts/tests/run.shindex ff2f1ac627..3bde1390d5 100755--- a/scripts/tests/run.sh+++ b/scripts/tests/run.sh@@ -16,7 +16,8 @@ Usage: scripts/tests/run.sh [test-script ...]Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subsetwhen specified. Documentation CI discovers every documentation suite and runsone of four disjoint, sorted shards; it does not maintain a second suite list.-Smoke CI partitions the complete inventory into two isolated, sorted shards.+Smoke CI balances the complete inventory across two isolated, sorted shards+using measured suite durations. New suites receive a default scheduling weight.EOF}@@ -134,10 +135,23 @@ main() {if ! discover_tests smoke_tests; thenreturn 1fi- local index- for ((index = $2; index < ${#smoke_tests[@]}; index += 2)); do- tests+=("${smoke_tests[index]}")- done+ local selection+ if (( ${#smoke_tests[@]} > 0 )); then+ # Capture the entire successful selector before admitting any suite. A+ # partial output from a failed selector is not a complete shard. Timing+ # hints only place discovered suites; they cannot add or omit coverage.+ if ! selection="$(+ printf '%s\n' "${smoke_tests[@]}" | python3 \+ "${ROOT_DIR}/scripts/select-smoke-test-shard.py" \+ "${ROOT_DIR}/.github/scripts/smoke-test-seconds.txt" 2 "$2"+ )"; then+ echo "Failed to partition the complete smoke test inventory" >&2+ return 1+ fi+ if [[ -n "${selection}" ]]; then+ mapfile -t tests <<< "${selection}"+ fi+ fiprintf 'Smoke shard %s/2: %s of %s suites\n' "$2" "${#tests[@]}" "${#smoke_tests[@]}"elif [[ $# -gt 0 ]]; thenlocal arg resolveddiff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.pyindex 7710cac481..855ac552f8 100644--- a/scripts/tests/test_smoke_runner.py+++ b/scripts/tests/test_smoke_runner.py@@ -26,6 +26,11 @@ class SmokeRunnerTest(unittest.TestCase):self.runner = self.tests / "run.sh"shutil.copyfile(RUNNER, self.runner)self.runner.chmod(0o700)+ self.selector = self.root / "scripts/select-smoke-test-shard.py"+ shutil.copyfile(ROOT / "scripts/select-smoke-test-shard.py", self.selector)+ self.weights = self.root / ".github/scripts/smoke-test-seconds.txt"+ self.weights.parent.mkdir(parents=True)+ self.weights.write_text("DEFAULT_WEIGHT 1\n", encoding="utf-8")self.bin = self.root / "bin"self.bin.mkdir()self.calls = self.root / "calls"@@ -298,5 +303,122 @@ class SmokeRunnerTest(unittest.TestCase):self.assertEqual(self.observed_calls(), [])+ def test_measured_weights_separate_slow_suites_without_changing_commands(self):+ names = [f"test_{index}.py" for index in range(8)]+ seconds = {name: 100 if index % 2 else 1 for index, name in enumerate(names)}+ for name in names:+ self.python_fixture(name)+ self.weights.write_text("DEFAULT_WEIGHT 1\n" + "".join(+ f"{name} {value}\n" for name, value in seconds.items()))+ seen = []+ for shard in range(2):+ before = len(seen)+ result = self.run_runner("--smoke-shard", shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ calls = self.observed_calls()[before:]+ seen.extend(calls)+ self.assertEqual(calls, sorted(calls))+ self.assertEqual(sum(seconds[name] for name in calls), 202)+ self.assertEqual(sorted(seen), names)+ self.assertEqual(len(seen), len(set(seen)))++ def test_unknown_renamed_and_stale_hints_never_change_discovered_coverage(self):+ names = ["test_a.py", "test_new.py", "test_renamed.py", "test-z.sh"]+ for name in names[:-1]:+ self.python_fixture(name)+ self.shell_fixture(names[-1])+ self.weights.write_text("DEFAULT_WEIGHT 2.5\ntest_a.py 20\ntest_removed.py 9999\n")+ for shard in range(2):+ result = self.run_runner("--smoke-shard", shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ calls = self.observed_calls()+ self.assertEqual(sorted(calls), sorted(names))+ self.assertEqual(len(calls), len(set(calls)))++ def test_weighted_shards_still_collect_every_failure_and_later_suite(self):+ names = [f"test_{index}.py" for index in range(8)]+ for index, name in enumerate(names):+ self.python_fixture(name, 7 if index in (1, 3) else 0)+ self.weights.write_text("DEFAULT_WEIGHT 1\n" + "".join(+ f"{name} {100 if index % 2 else 1}\n" for index, name in enumerate(names)))+ for shard in range(2):+ result = self.run_runner("--smoke-shard", shard)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Summary: 3/4 passed", result.stdout)+ self.assertIn("Failures: 1", result.stdout)+ self.assertEqual(sorted(self.observed_calls()), names)++ def test_failed_selector_cannot_admit_a_partial_passing_shard(self):+ passing = self.python_fixture("test_a.py")+ self.python_fixture("test_b.py", 7)+ self.selector.write_text(f"print({str(passing)!r})\nraise SystemExit(23)\n")+ result = self.run_runner("--smoke-shard", 0)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Failed to partition the complete smoke test inventory", result.stderr)+ self.assertNotIn("Summary:", result.stdout)+ self.assertEqual(self.observed_calls(), [])++ def test_missing_or_invalid_weights_reject_before_any_execution(self):+ self.python_fixture("test_first.py")+ cases = [None, "", "test_first.py 1\n", "DEFAULT_WEIGHT 1\nDEFAULT_WEIGHT 2\n",+ "DEFAULT_WEIGHT 1\ntest_first.py 1\ntest_first.py 2\n",+ "DEFAULT_WEIGHT 1\n../test_first.py 1\n", "DEFAULT_WEIGHT 1\ninvalid row extra\n"]+ cases.extend(f"DEFAULT_WEIGHT {value}\n" for value in ("nan", "Infinity", "0", "-1", "bad"))+ cases.extend(f"DEFAULT_WEIGHT 1\ntest_first.py {value}\n"+ for value in ("nan", "Infinity", "0", "-1", "bad"))+ for contents in cases:+ with self.subTest(contents=contents):+ if contents is None:+ self.weights.unlink()+ else:+ self.weights.write_text(contents)+ result = self.run_runner("--smoke-shard", 0)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Failed to partition the complete smoke test inventory", result.stderr)+ self.assertNotIn("Summary:", result.stdout)+ self.assertEqual(self.observed_calls(), [])++ def test_unsharded_explicit_and_docs_modes_do_not_consume_smoke_weights(self):+ self.python_fixture("test_first_docs.py")+ self.weights.unlink()+ self.selector.unlink()+ for arguments in ((), ("test_first_docs.py",), ("--docs-shard", "0")):+ with self.subTest(arguments=arguments):+ result = self.run_runner(*arguments)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertIn("Summary: 1/1 passed", result.stdout)++ def test_checked_in_hints_partition_complete_real_inventory_once(self):+ tests = sorted(str(path) for path in (ROOT / "scripts/tests").iterdir()+ if path.is_file() and (path.match("test-*.sh") or path.match("test_*.py")))+ seen = []+ for shard in range(2):+ result = subprocess.run(+ ["python3", str(ROOT / "scripts/select-smoke-test-shard.py"),+ str(ROOT / ".github/scripts/smoke-test-seconds.txt"), "2", str(shard)],+ input="\n".join(tests) + "\n", capture_output=True, text=True, timeout=15)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ selected = result.stdout.splitlines()+ self.assertTrue(selected)+ self.assertEqual(selected, sorted(selected))+ seen.extend(selected)+ self.assertEqual(sorted(seen), tests)+ self.assertEqual(len(seen), len(set(seen)))++ def test_selector_rejects_invalid_or_duplicate_inventory_and_shard_numbers(self):+ for count, index, inventory in [(2, 0, "test_a.py\ntest_a.py\n"),+ (2, 0, "not_a_test.py\n"),+ (2, 0, "test_a.py\n\ntest_b.py\n"),+ (0, 0, "test_a.py\n"),+ (2, 2, "test_a.py\n"),+ (2, -1, "test_a.py\n")]:+ with self.subTest(count=count, index=index, inventory=inventory):+ result = subprocess.run(+ ["python3", str(self.selector), str(self.weights), str(count), str(index)],+ input=inventory, capture_output=True, text=True, timeout=15)+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(result.stdout, "")++if __name__ == "__main__":unittest.main()