Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Balance isolated smoke suites with measured scheduling hints

pulse-triage[bot]db76f62798d8
diff --git a/.github/scripts/smoke-test-seconds.txt b/.github/scripts/smoke-test-seconds.txt
new file mode 100644
index 0000000000..014e76e242
--- /dev/null
+++ b/.github/scripts/smoke-test-seconds.txt
@@ -0,0 +1,22 @@
+# Script smoke suite seconds in ordinary PR CI, 2026-10-26.
+# Complete successful run 40313223658 at a5659362bf4b7466d2f308444bdd21849aeceb4c;
+# jobs 121540828495 / 121540828581, measured between suite headers.
+# These are scheduling hints, never the suite inventory or deadlines.
+# The current discovered tree supplies every suite; stale hints cannot add one.
+# Suites below 15 seconds use their measured mean, as do new/unmeasured suites.
+# Refresh from complete passing shard logs if imbalance returns.
+DEFAULT_WEIGHT 1.881
+test_admin_docs.py 25.874
+test_agent_log_docs.py 16.473
+test_api_failure_index.py 15.110
+test_authentication_diagnostic_docs.py 21.495
+test_auto_update_service_state.py 28.879
+test_docker_troubleshooting_docs.py 15.349
+test_memory_troubleshooting_docs.py 43.941
+test_notification_troubleshooting_docs.py 24.617
+test_npm_audit_retry.py 30.170
+test_performance_troubleshooting_docs.py 248.554
+test_server_installer_stop.py 38.558
+test_troubleshooting_logs.py 59.703
+test_truenas_docs.py 27.603
+test_vm_disk_diagnostics.py 80.059
diff --git a/scripts/select-smoke-test-shard.py b/scripts/select-smoke-test-shard.py
new file mode 100644
index 0000000000..ebda6869f7
--- /dev/null
+++ b/scripts/select-smoke-test-shard.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Balance whole smoke suites; discovery, never timing hints, defines coverage."""
+from __future__ import annotations
+
+import argparse
+from decimal import Decimal, InvalidOperation
+from pathlib import Path
+import sys
+
+
+def suite_name(name: str) -> bool:
+ return Path(name).name == name and (
+ name.startswith("test_") and name.endswith(".py")
+ or name.startswith("test-") and name.endswith(".sh")
+ )
+
+
+def read_weights(path: Path) -> tuple[Decimal, dict[str, Decimal]]:
+ default = None
+ weights: dict[str, Decimal] = {}
+ for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ line = raw.split("#", 1)[0].strip()
+ if not line:
+ continue
+ fields = line.split()
+ if len(fields) != 2:
+ raise ValueError(f"invalid timing hint at line {number}")
+ name, value = fields
+ try:
+ seconds = Decimal(value)
+ except InvalidOperation as error:
+ raise ValueError(f"invalid timing value at line {number}") from error
+ if not seconds.is_finite() or seconds <= 0:
+ raise ValueError(f"timing hint must be finite and positive at line {number}")
+ if name == "DEFAULT_WEIGHT":
+ if default is not None:
+ raise ValueError("duplicate DEFAULT_WEIGHT")
+ default = seconds
+ else:
+ if not suite_name(name) or name in weights:
+ raise ValueError(f"invalid or duplicate suite timing at line {number}")
+ weights[name] = seconds
+ if default is None:
+ raise ValueError("missing DEFAULT_WEIGHT")
+ return default, weights
+
+
+def select(paths: list[str], default: Decimal, weights: dict[str, Decimal],
+ count: int, index: int) -> list[str]:
+ if not 1 <= count <= 64 or not 0 <= index < count:
+ raise ValueError("invalid smoke shard count or index")
+ if len(paths) != len(set(paths)) or any(
+ not path or "\0" in path or not suite_name(Path(path).name) for path in paths
+ ):
+ raise ValueError("invalid or duplicate discovered smoke suite")
+ loads = [Decimal(0) for _ in range(count)]
+ shards: list[list[str]] = [[] for _ in range(count)]
+ # Stable ties keep unmeasured suites balanced too. We never iterate hints
+ # to construct the inventory, so new, renamed and removed suites are safe.
+ for path in sorted(paths, key=lambda item: (-weights.get(Path(item).name, default), item)):
+ shard = min(range(count), key=lambda candidate: (loads[candidate], candidate))
+ shards[shard].append(path)
+ loads[shard] += weights.get(Path(path).name, default)
+ # Retain the runner's sorted execution within each isolated shard. No
+ # suite's command, deadline, process cleanup or failure handling changes.
+ return sorted(shards[index])
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("weights", type=Path)
+ parser.add_argument("count", type=int)
+ parser.add_argument("index", type=int)
+ args = parser.parse_args()
+ try:
+ default, weights = read_weights(args.weights)
+ selected = select(sys.stdin.read().splitlines(), default, weights, args.count, args.index)
+ except (OSError, UnicodeError, ValueError) as error:
+ print(f"Cannot partition complete smoke inventory: {error}", file=sys.stderr)
+ return 1
+ if selected:
+ print("\n".join(selected))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index ff2f1ac627..3bde1390d5 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -16,7 +16,8 @@ Usage: scripts/tests/run.sh [test-script ...]
Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subset
when specified. Documentation CI discovers every documentation suite and runs
one of four disjoint, sorted shards; it does not maintain a second suite list.
-Smoke CI partitions the complete inventory into two isolated, sorted shards.
+Smoke CI balances the complete inventory across two isolated, sorted shards
+using measured suite durations. New suites receive a default scheduling weight.
EOF
}
@@ -134,10 +135,23 @@ main() {
if ! discover_tests smoke_tests; then
return 1
fi
- local index
- for ((index = $2; index < ${#smoke_tests[@]}; index += 2)); do
- tests+=("${smoke_tests[index]}")
- done
+ local selection
+ if (( ${#smoke_tests[@]} > 0 )); then
+ # Capture the entire successful selector before admitting any suite. A
+ # partial output from a failed selector is not a complete shard. Timing
+ # hints only place discovered suites; they cannot add or omit coverage.
+ if ! selection="$(
+ printf '%s\n' "${smoke_tests[@]}" | python3 \
+ "${ROOT_DIR}/scripts/select-smoke-test-shard.py" \
+ "${ROOT_DIR}/.github/scripts/smoke-test-seconds.txt" 2 "$2"
+ )"; then
+ echo "Failed to partition the complete smoke test inventory" >&2
+ return 1
+ fi
+ if [[ -n "${selection}" ]]; then
+ mapfile -t tests <<< "${selection}"
+ fi
+ fi
printf 'Smoke shard %s/2: %s of %s suites\n' "$2" "${#tests[@]}" "${#smoke_tests[@]}"
elif [[ $# -gt 0 ]]; then
local arg resolved
diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.py
index 7710cac481..855ac552f8 100644
--- a/scripts/tests/test_smoke_runner.py
+++ b/scripts/tests/test_smoke_runner.py
@@ -26,6 +26,11 @@ class SmokeRunnerTest(unittest.TestCase):
self.runner = self.tests / "run.sh"
shutil.copyfile(RUNNER, self.runner)
self.runner.chmod(0o700)
+ self.selector = self.root / "scripts/select-smoke-test-shard.py"
+ shutil.copyfile(ROOT / "scripts/select-smoke-test-shard.py", self.selector)
+ self.weights = self.root / ".github/scripts/smoke-test-seconds.txt"
+ self.weights.parent.mkdir(parents=True)
+ self.weights.write_text("DEFAULT_WEIGHT 1\n", encoding="utf-8")
self.bin = self.root / "bin"
self.bin.mkdir()
self.calls = self.root / "calls"
@@ -298,5 +303,122 @@ class SmokeRunnerTest(unittest.TestCase):
self.assertEqual(self.observed_calls(), [])
+ def test_measured_weights_separate_slow_suites_without_changing_commands(self):
+ names = [f"test_{index}.py" for index in range(8)]
+ seconds = {name: 100 if index % 2 else 1 for index, name in enumerate(names)}
+ for name in names:
+ self.python_fixture(name)
+ self.weights.write_text("DEFAULT_WEIGHT 1\n" + "".join(
+ f"{name} {value}\n" for name, value in seconds.items()))
+ seen = []
+ for shard in range(2):
+ before = len(seen)
+ result = self.run_runner("--smoke-shard", shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ calls = self.observed_calls()[before:]
+ seen.extend(calls)
+ self.assertEqual(calls, sorted(calls))
+ self.assertEqual(sum(seconds[name] for name in calls), 202)
+ self.assertEqual(sorted(seen), names)
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_unknown_renamed_and_stale_hints_never_change_discovered_coverage(self):
+ names = ["test_a.py", "test_new.py", "test_renamed.py", "test-z.sh"]
+ for name in names[:-1]:
+ self.python_fixture(name)
+ self.shell_fixture(names[-1])
+ self.weights.write_text("DEFAULT_WEIGHT 2.5\ntest_a.py 20\ntest_removed.py 9999\n")
+ for shard in range(2):
+ result = self.run_runner("--smoke-shard", shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ calls = self.observed_calls()
+ self.assertEqual(sorted(calls), sorted(names))
+ self.assertEqual(len(calls), len(set(calls)))
+
+ def test_weighted_shards_still_collect_every_failure_and_later_suite(self):
+ names = [f"test_{index}.py" for index in range(8)]
+ for index, name in enumerate(names):
+ self.python_fixture(name, 7 if index in (1, 3) else 0)
+ self.weights.write_text("DEFAULT_WEIGHT 1\n" + "".join(
+ f"{name} {100 if index % 2 else 1}\n" for index, name in enumerate(names)))
+ for shard in range(2):
+ result = self.run_runner("--smoke-shard", shard)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Summary: 3/4 passed", result.stdout)
+ self.assertIn("Failures: 1", result.stdout)
+ self.assertEqual(sorted(self.observed_calls()), names)
+
+ def test_failed_selector_cannot_admit_a_partial_passing_shard(self):
+ passing = self.python_fixture("test_a.py")
+ self.python_fixture("test_b.py", 7)
+ self.selector.write_text(f"print({str(passing)!r})\nraise SystemExit(23)\n")
+ result = self.run_runner("--smoke-shard", 0)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Failed to partition the complete smoke test inventory", result.stderr)
+ self.assertNotIn("Summary:", result.stdout)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_missing_or_invalid_weights_reject_before_any_execution(self):
+ self.python_fixture("test_first.py")
+ cases = [None, "", "test_first.py 1\n", "DEFAULT_WEIGHT 1\nDEFAULT_WEIGHT 2\n",
+ "DEFAULT_WEIGHT 1\ntest_first.py 1\ntest_first.py 2\n",
+ "DEFAULT_WEIGHT 1\n../test_first.py 1\n", "DEFAULT_WEIGHT 1\ninvalid row extra\n"]
+ cases.extend(f"DEFAULT_WEIGHT {value}\n" for value in ("nan", "Infinity", "0", "-1", "bad"))
+ cases.extend(f"DEFAULT_WEIGHT 1\ntest_first.py {value}\n"
+ for value in ("nan", "Infinity", "0", "-1", "bad"))
+ for contents in cases:
+ with self.subTest(contents=contents):
+ if contents is None:
+ self.weights.unlink()
+ else:
+ self.weights.write_text(contents)
+ result = self.run_runner("--smoke-shard", 0)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Failed to partition the complete smoke test inventory", result.stderr)
+ self.assertNotIn("Summary:", result.stdout)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_unsharded_explicit_and_docs_modes_do_not_consume_smoke_weights(self):
+ self.python_fixture("test_first_docs.py")
+ self.weights.unlink()
+ self.selector.unlink()
+ for arguments in ((), ("test_first_docs.py",), ("--docs-shard", "0")):
+ with self.subTest(arguments=arguments):
+ result = self.run_runner(*arguments)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertIn("Summary: 1/1 passed", result.stdout)
+
+ def test_checked_in_hints_partition_complete_real_inventory_once(self):
+ tests = sorted(str(path) for path in (ROOT / "scripts/tests").iterdir()
+ if path.is_file() and (path.match("test-*.sh") or path.match("test_*.py")))
+ seen = []
+ for shard in range(2):
+ result = subprocess.run(
+ ["python3", str(ROOT / "scripts/select-smoke-test-shard.py"),
+ str(ROOT / ".github/scripts/smoke-test-seconds.txt"), "2", str(shard)],
+ input="\n".join(tests) + "\n", capture_output=True, text=True, timeout=15)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ selected = result.stdout.splitlines()
+ self.assertTrue(selected)
+ self.assertEqual(selected, sorted(selected))
+ seen.extend(selected)
+ self.assertEqual(sorted(seen), tests)
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_selector_rejects_invalid_or_duplicate_inventory_and_shard_numbers(self):
+ for count, index, inventory in [(2, 0, "test_a.py\ntest_a.py\n"),
+ (2, 0, "not_a_test.py\n"),
+ (2, 0, "test_a.py\n\ntest_b.py\n"),
+ (0, 0, "test_a.py\n"),
+ (2, 2, "test_a.py\n"),
+ (2, -1, "test_a.py\n")]:
+ with self.subTest(count=count, index=index, inventory=inventory):
+ result = subprocess.run(
+ ["python3", str(self.selector), str(self.weights), str(count), str(index)],
+ input=inventory, capture_output=True, text=True, timeout=15)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(result.stdout, "")
+
+
if __name__ == "__main__":
unittest.main()