Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Balance complete documentation safety CI with measured suites
pulse-triage[bot]959f92a5a5ea
diff --git a/.github/scripts/docs-test-seconds.txt b/.github/scripts/docs-test-seconds.txtnew file mode 100644index 0000000000..b8ddc6a019--- /dev/null+++ b/.github/scripts/docs-test-seconds.txt@@ -0,0 +1,21 @@+# Documentation suite seconds in complete successful main CI, 2026-10-30.+# Run 41109701017 at 387249c8ad0ef3562db979cb9819a45d01d3fcfb;+# jobs 124248628165 / 124248628457 / 124248628657 / 124248628615.+# Measured between suite headers, including the last suite through Summary.+# Scheduling hints only: discovery defines coverage, never these names.+# Suites under 5s and new/unmeasured suites use the measured under-5s mean.+# Refresh from complete passing docs logs if imbalance returns.+DEFAULT_WEIGHT 0.539578+test_admin_docs.py 10.442204+test_agent_log_docs.py 16.497535+test_authentication_diagnostic_docs.py 14.153713+test_docker_troubleshooting_docs.py 15.532539+test_icmp_privilege_docs.py 8.267716+test_memory_troubleshooting_docs.py 28.289320+test_notification_troubleshooting_docs.py 25.115456+test_performance_troubleshooting_docs.py 92.960586+test_public_docs_workflow.py 26.553607+test_server_update_containment_docs.py 6.773839+test_troubleshooting_logs.py 37.678946+test_truenas_docs.py 25.534737+test_vm_disk_diagnostics.py 40.923979diff --git a/.github/workflows/public-docs.yml b/.github/workflows/public-docs.ymlindex c5ef3af498..c792660d12 100644--- a/.github/workflows/public-docs.yml+++ b/.github/workflows/public-docs.yml@@ -12,6 +12,8 @@ on:- "scripts/check_docs_mirror.py"- "scripts/readme_entrypoints_test.py"- "scripts/tests/**"+ - "scripts/select-smoke-test-shard.py"+ - ".github/scripts/docs-test-seconds.txt"- "scripts/test-vm-disk.sh"- ".github/workflows/public-docs.yml"push:@@ -27,6 +29,8 @@ on:- "scripts/check_docs_mirror.py"- "scripts/readme_entrypoints_test.py"- "scripts/tests/**"+ - "scripts/select-smoke-test-shard.py"+ - ".github/scripts/docs-test-seconds.txt"- "scripts/test-vm-disk.sh"- ".github/workflows/public-docs.yml"@@ -54,7 +58,8 @@ jobs:# Build and Test skips smoke for docs-only changes. Discover the complete# current docs suite here, including privacy, authentication and recovery.- # Isolated shards retain all recipe deadlines without a longer CI path.+ # Measured whole-suite scheduling retains sorted, isolated execution and+ # every recipe deadline; timing hints cannot add or omit coverage.- name: Exercise documentation safety and diagnostic recipesrun: scripts/tests/run.sh --docs-shard "${{ matrix.shard }}"diff --git a/scripts/tests/run.sh b/scripts/tests/run.shindex 3bde1390d5..6a2482ca8c 100755--- a/scripts/tests/run.sh+++ b/scripts/tests/run.sh@@ -14,10 +14,10 @@ Usage: scripts/tests/run.sh [test-script ...]scripts/tests/run.sh --docs-shard <0|1|2|3>Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subset-when specified. Documentation CI discovers every documentation suite and runs-one of four disjoint, sorted shards; it does not maintain a second suite list.-Smoke CI balances the complete inventory across two isolated, sorted shards-using measured suite durations. New suites receive a default scheduling weight.+when specified. Documentation and smoke CI balance their complete discovered+inventories across four and two isolated, sorted shards respectively, using+measured suite durations. Hints never define coverage or change test deadlines;+new suites receive a default scheduling weight.EOF}@@ -121,10 +121,22 @@ main() {if ! discover_tests doc_tests docs; thenreturn 1fi- local index- for ((index = $2; index < ${#doc_tests[@]}; index += 4)); do- tests+=("${doc_tests[index]}")- done+ local doc_selection+ if (( ${#doc_tests[@]} > 0 )); then+ # Reuse smoke's whole-suite partitioner, with docs-only measurements.+ # Discovery remains the inventory; admit no partial selector output.+ if ! doc_selection="$(+ printf '%s\n' "${doc_tests[@]}" | python3 \+ "${ROOT_DIR}/scripts/select-smoke-test-shard.py" \+ "${ROOT_DIR}/.github/scripts/docs-test-seconds.txt" 4 "$2"+ )"; then+ echo "Failed to partition the complete documentation test inventory" >&2+ return 1+ fi+ if [[ -n "${doc_selection}" ]]; then+ mapfile -t tests <<< "${doc_selection}"+ fi+ fiprintf 'Documentation shard %s/4: %s of %s suites\n' "$2" "${#tests[@]}" "${#doc_tests[@]}"elif [[ "${1:-}" == --smoke-shard ]]; thenif [[ $# -ne 2 || ! "${2:-}" =~ ^[0-1]$ ]]; thendiff --git a/scripts/tests/test_public_docs_workflow.py b/scripts/tests/test_public_docs_workflow.pyindex 411f722e17..85133b2905 100644--- a/scripts/tests/test_public_docs_workflow.py+++ b/scripts/tests/test_public_docs_workflow.py@@ -89,6 +89,7 @@ class PublicDocsWorkflowTest(unittest.TestCase):".github/ISSUE_TEMPLATE/**", ".github/PULL_REQUEST_TEMPLATE.md","scripts/check_public_docs.py", "scripts/check_docs_mirror.py","scripts/readme_entrypoints_test.py", "scripts/tests/**",+ "scripts/select-smoke-test-shard.py", ".github/scripts/docs-test-seconds.txt","scripts/test-vm-disk.sh", ".github/workflows/public-docs.yml",}for event in ("pull_request", "push"):@@ -153,6 +154,11 @@ class PublicDocsWorkflowTest(unittest.TestCase):tests.mkdir(parents=True)shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")(tests / "run.sh").chmod(0o700)+ shutil.copyfile(ROOT / "scripts/select-smoke-test-shard.py",+ root / "scripts/select-smoke-test-shard.py")+ hints = root / ".github/scripts/docs-test-seconds.txt"+ hints.parent.mkdir(parents=True)+ shutil.copyfile(ROOT / ".github/scripts/docs-test-seconds.txt", hints)for name in documentation_suites():status = 1 if name == failing else 0if name.endswith(".sh"):@@ -174,21 +180,31 @@ class PublicDocsWorkflowTest(unittest.TestCase):for shard in range(4):result, calls = self.run_adapters(shard)self.assertEqual(result.returncode, 0, result.stdout + result.stderr)- self.assertEqual(calls, documentation_suites()[shard::4])+ self.assertTrue(calls)+ self.assertEqual(calls, sorted(calls))self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)seen.extend(calls)self.assertEqual(sorted(seen), documentation_suites())self.assertEqual(len(seen), len(set(seen)))def test_each_failed_suite_fails_its_actual_workflow_command(self):- for index, failing in enumerate(documentation_suites()):- with self.subTest(failing=failing):- shard = index % 4- result, calls = self.run_adapters(shard, failing)- self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)- self.assertEqual(calls, documentation_suites()[shard::4])- self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)- self.assertIn("Failures: 1", result.stdout)+ seen = []+ for shard in range(4):+ # Observe actual membership, not obsolete filename parity. The+ # complete-union assertion independently catches missing coverage.+ selected, names = self.run_adapters(shard)+ self.assertEqual(selected.returncode, 0, selected.stdout + selected.stderr)+ self.assertTrue(names)+ for failing in names:+ with self.subTest(failing=failing, shard=shard):+ result, calls = self.run_adapters(shard, failing)+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(calls, names)+ self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)+ self.assertIn("Failures: 1", result.stdout)+ seen.append(failing)+ self.assertEqual(sorted(seen), documentation_suites())+ self.assertEqual(len(seen), len(set(seen)))if __name__ == "__main__":diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.pyindex 855ac552f8..374a49c113 100644--- a/scripts/tests/test_smoke_runner.py+++ b/scripts/tests/test_smoke_runner.py@@ -31,6 +31,8 @@ class SmokeRunnerTest(unittest.TestCase):self.weights = self.root / ".github/scripts/smoke-test-seconds.txt"self.weights.parent.mkdir(parents=True)self.weights.write_text("DEFAULT_WEIGHT 1\n", encoding="utf-8")+ self.doc_weights = self.weights.with_name("docs-test-seconds.txt")+ self.doc_weights.write_text("DEFAULT_WEIGHT 1\n", encoding="utf-8")self.bin = self.root / "bin"self.bin.mkdir()self.calls = self.root / "calls"@@ -378,16 +380,94 @@ class SmokeRunnerTest(unittest.TestCase):self.assertNotIn("Summary:", result.stdout)self.assertEqual(self.observed_calls(), [])- def test_unsharded_explicit_and_docs_modes_do_not_consume_smoke_weights(self):+ def test_unsharded_and_explicit_modes_need_no_selector_or_weights(self):self.python_fixture("test_first_docs.py")self.weights.unlink()+ self.doc_weights.unlink()self.selector.unlink()- for arguments in ((), ("test_first_docs.py",), ("--docs-shard", "0")):+ for arguments in ((), ("test_first_docs.py",)):with self.subTest(arguments=arguments):result = self.run_runner(*arguments)self.assertEqual(result.returncode, 0, result.stdout + result.stderr)self.assertIn("Summary: 1/1 passed", result.stdout)+ def test_each_sharded_mode_only_consumes_its_own_hints(self):+ self.python_fixture("test_first_docs.py")+ self.weights.write_text("invalid smoke hints\n")+ docs = self.run_runner("--docs-shard", 0)+ self.assertEqual(docs.returncode, 0, docs.stdout + docs.stderr)+ self.weights.write_text("DEFAULT_WEIGHT 1\n")+ self.doc_weights.write_text("invalid docs hints\n")+ smoke = self.run_runner("--smoke-shard", 0)+ self.assertEqual(smoke.returncode, 0, smoke.stdout + smoke.stderr)++ def test_doc_weights_separate_long_suites_without_changing_commands(self):+ names = [f"test_{index:02d}_docs.py" for index in range(16)]+ seconds = {name: 100 if index % 4 == 1 else 1 for index, name in enumerate(names)}+ for index, name in enumerate(names):+ # Each shard must retain an adverse verdict and all later suites.+ self.python_fixture(name, 7 if index % 4 == 1 else 0)+ self.doc_weights.write_text("DEFAULT_WEIGHT 1\n" + "".join(+ f"{name} {value}\n" for name, value in seconds.items()))+ seen = []+ for shard in range(4):+ before = len(self.observed_calls())+ result = self.run_runner("--docs-shard", shard)+ calls = self.observed_calls()[before:]+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(calls, sorted(calls))+ self.assertEqual(sum(seconds[name] for name in calls), 103)+ self.assertIn("Summary: 3/4 passed", result.stdout)+ self.assertIn("Failures: 1", result.stdout)+ seen.extend(calls)+ self.assertEqual(sorted(seen), names)+ self.assertEqual(len(seen), len(set(seen)))++ def test_doc_hints_cannot_add_removed_or_omit_new_and_legacy_suites(self):+ names = ["test_a_docs.py", "test_new_docs.py", "test_renamed_docs.py",+ "test_vm_disk_diagnostics.py", "test_troubleshooting_logs.py"]+ for name in names:+ self.python_fixture(name)+ names.append("test-retired-trial-acquisition-docs.sh")+ self.shell_fixture(names[-1])+ self.doc_weights.write_text("DEFAULT_WEIGHT 2.5\ntest_a_docs.py 20\ntest_removed_docs.py 9999\n")+ for shard in range(4):+ result = self.run_runner("--docs-shard", shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ calls = self.observed_calls()+ self.assertEqual(sorted(calls), sorted(names))+ self.assertEqual(len(calls), len(set(calls)))++ def test_failed_doc_selector_cannot_admit_a_partial_passing_shard(self):+ passing = self.python_fixture("test_a_docs.py")+ self.python_fixture("test_b_docs.py", 7)+ self.selector.write_text(f"print({str(passing)!r})\nraise SystemExit(23)\n")+ result = self.run_runner("--docs-shard", 0)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Failed to partition the complete documentation test inventory", result.stderr)+ self.assertNotIn("Summary:", result.stdout)+ self.assertEqual(self.observed_calls(), [])++ def test_missing_or_invalid_doc_hints_reject_before_any_execution(self):+ self.python_fixture("test_first_docs.py")+ cases = [None, "", "test_first_docs.py 1\n", "DEFAULT_WEIGHT 1\nDEFAULT_WEIGHT 2\n",+ "DEFAULT_WEIGHT 1\ntest_first_docs.py 1\ntest_first_docs.py 2\n",+ "DEFAULT_WEIGHT 1\n../test_first_docs.py 1\n", "DEFAULT_WEIGHT 1\ninvalid row extra\n"]+ cases.extend(f"DEFAULT_WEIGHT {value}\n" for value in ("nan", "Infinity", "0", "-1", "bad"))+ cases.extend(f"DEFAULT_WEIGHT 1\ntest_first_docs.py {value}\n"+ for value in ("nan", "Infinity", "0", "-1", "bad"))+ for contents in cases:+ with self.subTest(contents=contents):+ if contents is None:+ self.doc_weights.unlink()+ else:+ self.doc_weights.write_text(contents)+ result = self.run_runner("--docs-shard", 0)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Failed to partition the complete documentation test inventory", result.stderr)+ self.assertNotIn("Summary:", result.stdout)+ self.assertEqual(self.observed_calls(), [])+def test_checked_in_hints_partition_complete_real_inventory_once(self):tests = sorted(str(path) for path in (ROOT / "scripts/tests").iterdir()if path.is_file() and (path.match("test-*.sh") or path.match("test_*.py")))