Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Balance complete documentation safety CI with measured suites

pulse-triage[bot]959f92a5a5ea
diff --git a/.github/scripts/docs-test-seconds.txt b/.github/scripts/docs-test-seconds.txt
new file mode 100644
index 0000000000..b8ddc6a019
--- /dev/null
+++ b/.github/scripts/docs-test-seconds.txt
@@ -0,0 +1,21 @@
+# Documentation suite seconds in complete successful main CI, 2026-10-30.
+# Run 41109701017 at 387249c8ad0ef3562db979cb9819a45d01d3fcfb;
+# jobs 124248628165 / 124248628457 / 124248628657 / 124248628615.
+# Measured between suite headers, including the last suite through Summary.
+# Scheduling hints only: discovery defines coverage, never these names.
+# Suites under 5s and new/unmeasured suites use the measured under-5s mean.
+# Refresh from complete passing docs logs if imbalance returns.
+DEFAULT_WEIGHT 0.539578
+test_admin_docs.py 10.442204
+test_agent_log_docs.py 16.497535
+test_authentication_diagnostic_docs.py 14.153713
+test_docker_troubleshooting_docs.py 15.532539
+test_icmp_privilege_docs.py 8.267716
+test_memory_troubleshooting_docs.py 28.289320
+test_notification_troubleshooting_docs.py 25.115456
+test_performance_troubleshooting_docs.py 92.960586
+test_public_docs_workflow.py 26.553607
+test_server_update_containment_docs.py 6.773839
+test_troubleshooting_logs.py 37.678946
+test_truenas_docs.py 25.534737
+test_vm_disk_diagnostics.py 40.923979
diff --git a/.github/workflows/public-docs.yml b/.github/workflows/public-docs.yml
index c5ef3af498..c792660d12 100644
--- a/.github/workflows/public-docs.yml
+++ b/.github/workflows/public-docs.yml
@@ -12,6 +12,8 @@ on:
- "scripts/check_docs_mirror.py"
- "scripts/readme_entrypoints_test.py"
- "scripts/tests/**"
+ - "scripts/select-smoke-test-shard.py"
+ - ".github/scripts/docs-test-seconds.txt"
- "scripts/test-vm-disk.sh"
- ".github/workflows/public-docs.yml"
push:
@@ -27,6 +29,8 @@ on:
- "scripts/check_docs_mirror.py"
- "scripts/readme_entrypoints_test.py"
- "scripts/tests/**"
+ - "scripts/select-smoke-test-shard.py"
+ - ".github/scripts/docs-test-seconds.txt"
- "scripts/test-vm-disk.sh"
- ".github/workflows/public-docs.yml"
@@ -54,7 +58,8 @@ jobs:
# Build and Test skips smoke for docs-only changes. Discover the complete
# current docs suite here, including privacy, authentication and recovery.
- # Isolated shards retain all recipe deadlines without a longer CI path.
+ # Measured whole-suite scheduling retains sorted, isolated execution and
+ # every recipe deadline; timing hints cannot add or omit coverage.
- name: Exercise documentation safety and diagnostic recipes
run: scripts/tests/run.sh --docs-shard "${{ matrix.shard }}"
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index 3bde1390d5..6a2482ca8c 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -14,10 +14,10 @@ Usage: scripts/tests/run.sh [test-script ...]
scripts/tests/run.sh --docs-shard <0|1|2|3>
Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subset
-when specified. Documentation CI discovers every documentation suite and runs
-one of four disjoint, sorted shards; it does not maintain a second suite list.
-Smoke CI balances the complete inventory across two isolated, sorted shards
-using measured suite durations. New suites receive a default scheduling weight.
+when specified. Documentation and smoke CI balance their complete discovered
+inventories across four and two isolated, sorted shards respectively, using
+measured suite durations. Hints never define coverage or change test deadlines;
+new suites receive a default scheduling weight.
EOF
}
@@ -121,10 +121,22 @@ main() {
if ! discover_tests doc_tests docs; then
return 1
fi
- local index
- for ((index = $2; index < ${#doc_tests[@]}; index += 4)); do
- tests+=("${doc_tests[index]}")
- done
+ local doc_selection
+ if (( ${#doc_tests[@]} > 0 )); then
+ # Reuse smoke's whole-suite partitioner, with docs-only measurements.
+ # Discovery remains the inventory; admit no partial selector output.
+ if ! doc_selection="$(
+ printf '%s\n' "${doc_tests[@]}" | python3 \
+ "${ROOT_DIR}/scripts/select-smoke-test-shard.py" \
+ "${ROOT_DIR}/.github/scripts/docs-test-seconds.txt" 4 "$2"
+ )"; then
+ echo "Failed to partition the complete documentation test inventory" >&2
+ return 1
+ fi
+ if [[ -n "${doc_selection}" ]]; then
+ mapfile -t tests <<< "${doc_selection}"
+ fi
+ fi
printf 'Documentation shard %s/4: %s of %s suites\n' "$2" "${#tests[@]}" "${#doc_tests[@]}"
elif [[ "${1:-}" == --smoke-shard ]]; then
if [[ $# -ne 2 || ! "${2:-}" =~ ^[0-1]$ ]]; then
diff --git a/scripts/tests/test_public_docs_workflow.py b/scripts/tests/test_public_docs_workflow.py
index 411f722e17..85133b2905 100644
--- a/scripts/tests/test_public_docs_workflow.py
+++ b/scripts/tests/test_public_docs_workflow.py
@@ -89,6 +89,7 @@ class PublicDocsWorkflowTest(unittest.TestCase):
".github/ISSUE_TEMPLATE/**", ".github/PULL_REQUEST_TEMPLATE.md",
"scripts/check_public_docs.py", "scripts/check_docs_mirror.py",
"scripts/readme_entrypoints_test.py", "scripts/tests/**",
+ "scripts/select-smoke-test-shard.py", ".github/scripts/docs-test-seconds.txt",
"scripts/test-vm-disk.sh", ".github/workflows/public-docs.yml",
}
for event in ("pull_request", "push"):
@@ -153,6 +154,11 @@ class PublicDocsWorkflowTest(unittest.TestCase):
tests.mkdir(parents=True)
shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")
(tests / "run.sh").chmod(0o700)
+ shutil.copyfile(ROOT / "scripts/select-smoke-test-shard.py",
+ root / "scripts/select-smoke-test-shard.py")
+ hints = root / ".github/scripts/docs-test-seconds.txt"
+ hints.parent.mkdir(parents=True)
+ shutil.copyfile(ROOT / ".github/scripts/docs-test-seconds.txt", hints)
for name in documentation_suites():
status = 1 if name == failing else 0
if name.endswith(".sh"):
@@ -174,21 +180,31 @@ class PublicDocsWorkflowTest(unittest.TestCase):
for shard in range(4):
result, calls = self.run_adapters(shard)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
- self.assertEqual(calls, documentation_suites()[shard::4])
+ self.assertTrue(calls)
+ self.assertEqual(calls, sorted(calls))
self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)
seen.extend(calls)
self.assertEqual(sorted(seen), documentation_suites())
self.assertEqual(len(seen), len(set(seen)))
def test_each_failed_suite_fails_its_actual_workflow_command(self):
- for index, failing in enumerate(documentation_suites()):
- with self.subTest(failing=failing):
- shard = index % 4
- result, calls = self.run_adapters(shard, failing)
- self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
- self.assertEqual(calls, documentation_suites()[shard::4])
- self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)
- self.assertIn("Failures: 1", result.stdout)
+ seen = []
+ for shard in range(4):
+ # Observe actual membership, not obsolete filename parity. The
+ # complete-union assertion independently catches missing coverage.
+ selected, names = self.run_adapters(shard)
+ self.assertEqual(selected.returncode, 0, selected.stdout + selected.stderr)
+ self.assertTrue(names)
+ for failing in names:
+ with self.subTest(failing=failing, shard=shard):
+ result, calls = self.run_adapters(shard, failing)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(calls, names)
+ self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)
+ self.assertIn("Failures: 1", result.stdout)
+ seen.append(failing)
+ self.assertEqual(sorted(seen), documentation_suites())
+ self.assertEqual(len(seen), len(set(seen)))
if __name__ == "__main__":
diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.py
index 855ac552f8..374a49c113 100644
--- a/scripts/tests/test_smoke_runner.py
+++ b/scripts/tests/test_smoke_runner.py
@@ -31,6 +31,8 @@ class SmokeRunnerTest(unittest.TestCase):
self.weights = self.root / ".github/scripts/smoke-test-seconds.txt"
self.weights.parent.mkdir(parents=True)
self.weights.write_text("DEFAULT_WEIGHT 1\n", encoding="utf-8")
+ self.doc_weights = self.weights.with_name("docs-test-seconds.txt")
+ self.doc_weights.write_text("DEFAULT_WEIGHT 1\n", encoding="utf-8")
self.bin = self.root / "bin"
self.bin.mkdir()
self.calls = self.root / "calls"
@@ -378,16 +380,94 @@ class SmokeRunnerTest(unittest.TestCase):
self.assertNotIn("Summary:", result.stdout)
self.assertEqual(self.observed_calls(), [])
- def test_unsharded_explicit_and_docs_modes_do_not_consume_smoke_weights(self):
+ def test_unsharded_and_explicit_modes_need_no_selector_or_weights(self):
self.python_fixture("test_first_docs.py")
self.weights.unlink()
+ self.doc_weights.unlink()
self.selector.unlink()
- for arguments in ((), ("test_first_docs.py",), ("--docs-shard", "0")):
+ for arguments in ((), ("test_first_docs.py",)):
with self.subTest(arguments=arguments):
result = self.run_runner(*arguments)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("Summary: 1/1 passed", result.stdout)
+ def test_each_sharded_mode_only_consumes_its_own_hints(self):
+ self.python_fixture("test_first_docs.py")
+ self.weights.write_text("invalid smoke hints\n")
+ docs = self.run_runner("--docs-shard", 0)
+ self.assertEqual(docs.returncode, 0, docs.stdout + docs.stderr)
+ self.weights.write_text("DEFAULT_WEIGHT 1\n")
+ self.doc_weights.write_text("invalid docs hints\n")
+ smoke = self.run_runner("--smoke-shard", 0)
+ self.assertEqual(smoke.returncode, 0, smoke.stdout + smoke.stderr)
+
+ def test_doc_weights_separate_long_suites_without_changing_commands(self):
+ names = [f"test_{index:02d}_docs.py" for index in range(16)]
+ seconds = {name: 100 if index % 4 == 1 else 1 for index, name in enumerate(names)}
+ for index, name in enumerate(names):
+ # Each shard must retain an adverse verdict and all later suites.
+ self.python_fixture(name, 7 if index % 4 == 1 else 0)
+ self.doc_weights.write_text("DEFAULT_WEIGHT 1\n" + "".join(
+ f"{name} {value}\n" for name, value in seconds.items()))
+ seen = []
+ for shard in range(4):
+ before = len(self.observed_calls())
+ result = self.run_runner("--docs-shard", shard)
+ calls = self.observed_calls()[before:]
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(calls, sorted(calls))
+ self.assertEqual(sum(seconds[name] for name in calls), 103)
+ self.assertIn("Summary: 3/4 passed", result.stdout)
+ self.assertIn("Failures: 1", result.stdout)
+ seen.extend(calls)
+ self.assertEqual(sorted(seen), names)
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_doc_hints_cannot_add_removed_or_omit_new_and_legacy_suites(self):
+ names = ["test_a_docs.py", "test_new_docs.py", "test_renamed_docs.py",
+ "test_vm_disk_diagnostics.py", "test_troubleshooting_logs.py"]
+ for name in names:
+ self.python_fixture(name)
+ names.append("test-retired-trial-acquisition-docs.sh")
+ self.shell_fixture(names[-1])
+ self.doc_weights.write_text("DEFAULT_WEIGHT 2.5\ntest_a_docs.py 20\ntest_removed_docs.py 9999\n")
+ for shard in range(4):
+ result = self.run_runner("--docs-shard", shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ calls = self.observed_calls()
+ self.assertEqual(sorted(calls), sorted(names))
+ self.assertEqual(len(calls), len(set(calls)))
+
+ def test_failed_doc_selector_cannot_admit_a_partial_passing_shard(self):
+ passing = self.python_fixture("test_a_docs.py")
+ self.python_fixture("test_b_docs.py", 7)
+ self.selector.write_text(f"print({str(passing)!r})\nraise SystemExit(23)\n")
+ result = self.run_runner("--docs-shard", 0)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Failed to partition the complete documentation test inventory", result.stderr)
+ self.assertNotIn("Summary:", result.stdout)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_missing_or_invalid_doc_hints_reject_before_any_execution(self):
+ self.python_fixture("test_first_docs.py")
+ cases = [None, "", "test_first_docs.py 1\n", "DEFAULT_WEIGHT 1\nDEFAULT_WEIGHT 2\n",
+ "DEFAULT_WEIGHT 1\ntest_first_docs.py 1\ntest_first_docs.py 2\n",
+ "DEFAULT_WEIGHT 1\n../test_first_docs.py 1\n", "DEFAULT_WEIGHT 1\ninvalid row extra\n"]
+ cases.extend(f"DEFAULT_WEIGHT {value}\n" for value in ("nan", "Infinity", "0", "-1", "bad"))
+ cases.extend(f"DEFAULT_WEIGHT 1\ntest_first_docs.py {value}\n"
+ for value in ("nan", "Infinity", "0", "-1", "bad"))
+ for contents in cases:
+ with self.subTest(contents=contents):
+ if contents is None:
+ self.doc_weights.unlink()
+ else:
+ self.doc_weights.write_text(contents)
+ result = self.run_runner("--docs-shard", 0)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Failed to partition the complete documentation test inventory", result.stderr)
+ self.assertNotIn("Summary:", result.stdout)
+ self.assertEqual(self.observed_calls(), [])
+
def test_checked_in_hints_partition_complete_real_inventory_once(self):
tests = sorted(str(path) for path in (ROOT / "scripts/tests").iterdir()
if path.is_file() and (path.match("test-*.sh") or path.match("test_*.py")))