Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bound native fixture controls independently

rcourtman0b3261e1aa13
diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go
index 859b369e3c..ff6d33bd77 100644
--- a/scripts/installtests/install_sh_test.go
+++ b/scripts/installtests/install_sh_test.go
@@ -8599,14 +8599,7 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T)
if runtime.GOOS == "windows" {
t.Skip("TrueNAS SCALE shell controls; Windows has its own native SCM gate")
}
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
- defer cancel()
- cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_truenas_teardown.py"))
- output, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("TrueNAS SCALE boot-task ownership/retirement controls: %v\n%s", err, output)
- }
- for _, name := range []string{
+ runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_truenas_teardown.py"), "AgentTrueNASTeardown", []string{
"test_unavailable_or_untrusted_task_inventory_cannot_remove_recovery",
"test_foreign_or_indirect_bootstrap_is_not_mutated",
"test_missing_activation_target_cannot_hide_an_owned_task",
@@ -8623,29 +8616,14 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T)
"test_disabled_absent_missing_bootstrap_and_repeated_removal_are_observed",
"test_ordinary_linux_state_is_not_appliance_evidence",
"test_process_read_exposes_only_roles_and_real_timeout_child_is_stopped",
- } {
- if !strings.Contains(string(output), name+" (__main__.AgentTrueNASTeardown."+name+") ... ok") {
- t.Fatalf("TrueNAS SCALE control %s did not pass:\n%s", name, output)
- }
- }
- if !strings.Contains(string(output), "\nRan 16 tests in ") || !strings.Contains(string(output), "\nOK\n") {
- t.Fatalf("incomplete TrueNAS SCALE verdict:\n%s", output)
- }
- t.Logf("%s", output)
+ }, 2*time.Minute)
}
func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Unix NAS shell controls; Windows retains its separate native SCM gate")
}
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
- defer cancel()
- cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_nas_teardown.py"))
- output, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("NAS boot-owner/watchdog production caller controls: %v\n%s", err, output)
- }
- for _, name := range []string{
+ runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_nas_teardown.py"), "AgentNASTeardown", []string{
"test_complete_process_evidence_is_required_before_boot_or_remote_mutation",
"test_unknown_relative_foreign_or_deferred_boot_owners_are_not_signalled",
"test_non_regular_foreign_wrapper_and_unselected_state_preserve_recovery",
@@ -8669,13 +8647,5 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {
"test_private_process_reader_requests_native_basename_and_full_argv",
"test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats",
"test_caller_has_no_blanket_nas_or_legacy_tree_deletion",
- } {
- if !strings.Contains(string(output), name+" (__main__.AgentNASTeardown."+name+") ... ok") {
- t.Fatalf("NAS removal control %s did not pass:\n%s", name, output)
- }
- }
- if !strings.Contains(string(output), "\nRan 23 tests in ") || !strings.Contains(string(output), "\nOK\n") {
- t.Fatalf("incomplete NAS removal verdict:\n%s", output)
- }
- t.Logf("%s", output)
+ }, 2*time.Minute)
}
diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go
index 63a0f401bf..a147c00f44 100644
--- a/scripts/installtests/pulse_auto_update_test.go
+++ b/scripts/installtests/pulse_auto_update_test.go
@@ -162,18 +162,7 @@ func TestAutoUpdateDownloadFailures(t *testing.T) {
// Service-state admission is independent of the signed/metadata transaction
// matrix below; no native service or published input is used by this fixture.
func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
- defer cancel()
- cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateServiceState")
- cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
- out, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)
- }
- // Keep the manager/collector controls part of actual package acceptance, not just
- // an optional Python check. A green fixture with these methods removed
- // would otherwise silently lose early-failure and final-line coverage.
- required := []string{
+ runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateServiceState", []string{
"test_manager_diagnostics_refuse_initial_update_and_restart",
"test_manager_diagnostics_after_install_retain_backup_without_mutation",
"test_manager_diagnostics_after_stop_retain_backup_without_restart",
@@ -181,46 +170,17 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
"test_rollback_keeps_backup_until_restored_running_executable_matches",
"test_early_collector_failure_survives_later_success_and_drains_installer",
"test_unterminated_final_installer_line_is_collected_and_its_failure_retained",
- }
- for _, name := range required {
- passed := false
- for _, line := range strings.Split(string(out), "\n") {
- if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") {
- passed = true
- }
- }
- if !passed {
- t.Fatalf("required updater collector control %s did not pass:\n%s", name, out)
- }
- }
+ }, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
}
// Keep the full real-sink matrix mandatory, but give it its own existing-size
// bound rather than exhausting the service-state suite's manager deadlines.
func TestAutoUpdateLogSinkFailures(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
- defer cancel()
- cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateLogSinks")
- cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
- out, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("unattended real-log sink controls: %v\n%s", err, out)
- }
- for _, name := range []string{
+ runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateLogSinks", []string{
"test_logger_double_keeps_complete_argument_frames_and_exit_sequence",
"test_real_log_retains_each_sink_failure_in_conditional_calls",
"test_real_logger_failure_drains_installer_and_preserves_recovery",
- } {
- passed := false
- for _, line := range strings.Split(string(out), "\n") {
- if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") {
- passed = true
- }
- }
- if !passed {
- t.Fatalf("required real-log sink control %s did not pass:\n%s", name, out)
- }
- }
+ }, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
}
// A real owned process retains its old executable after its disk path changes.
diff --git a/scripts/installtests/python_fixture_controls_test.go b/scripts/installtests/python_fixture_controls_test.go
new file mode 100644
index 0000000000..e24a56321b
--- /dev/null
+++ b/scripts/installtests/python_fixture_controls_test.go
@@ -0,0 +1,123 @@
+package installtests
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+// A suite's cumulative runtime is not a service or installer deadline. Give
+// each independent control the existing bound, while the native phase keeps
+// its overall Go timeout. Discovery runs every method, including future ones,
+// and the required list prevents deleting a safety control unnoticed.
+func runPythonFixtureControls(t *testing.T, script, class string, required []string, bound time.Duration, env ...string) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), bound)
+ controls, err := discoverPythonFixtureControls(ctx, script, class, required, env...)
+ cancel()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("Discovered %d %s controls; each retains its %s bound", len(controls), class, bound)
+ for _, name := range controls {
+ t.Run(name, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), bound)
+ defer cancel()
+ output, err := executePythonFixtureControl(ctx, script, class, name, env...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("%s", output)
+ })
+ }
+}
+
+func discoverPythonFixtureControls(ctx context.Context, script, class string, required []string, env ...string) ([]string, error) {
+ const discover = `import json, runpy, sys, unittest
+namespace = runpy.run_path(sys.argv[1])
+suite = unittest.defaultTestLoader.loadTestsFromTestCase(namespace[sys.argv[2]])
+print(json.dumps([test._testMethodName for test in suite]))
+`
+ cmd := exec.CommandContext(ctx, "python3", "-I", "-S", "-c", discover, script, class)
+ cmd.Env = append(os.Environ(), env...)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return nil, fmt.Errorf("discover %s controls: %w (context %v)\n%s", class, err, ctx.Err(), output)
+ }
+ var controls []string
+ if err := json.Unmarshal(output, &controls); err != nil {
+ return nil, fmt.Errorf("decode %s controls: %w\n%s", class, err, output)
+ }
+ if len(controls) == 0 {
+ return nil, fmt.Errorf("no controls discovered in %s", class)
+ }
+ for _, name := range required {
+ found := false
+ for _, control := range controls {
+ found = found || control == name
+ }
+ if !found {
+ return nil, fmt.Errorf("required %s control %s is absent from discovery", class, name)
+ }
+ }
+ return controls, nil
+}
+
+func executePythonFixtureControl(ctx context.Context, script, class, name string, env ...string) ([]byte, error) {
+ selector := class + "." + name
+ cmd := exec.CommandContext(ctx, "python3", "-I", "-S", script, "-v", selector)
+ cmd.Env = append(os.Environ(), env...)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return output, fmt.Errorf("%s: %w (context %v)\n%s", selector, err, ctx.Err(), output)
+ }
+ if !strings.Contains(string(output), name+" (__main__."+selector+") ... ok") ||
+ !strings.Contains(string(output), "\nRan 1 test in ") ||
+ !strings.Contains(string(output), "\nOK\n") {
+ return output, fmt.Errorf("%s did not execute exactly one passing control:\n%s", selector, output)
+ }
+ return output, nil
+}
+
+func TestPythonFixtureControlsRetainDiscoveryAndRejectFalsePasses(t *testing.T) {
+ script := filepath.Join(t.TempDir(), "controls.py")
+ const source = `import unittest
+class Controls(unittest.TestCase):
+ def test_added(self): pass
+ def test_fail(self): self.fail("retained adverse control")
+ def test_pass(self): pass
+ @unittest.skip("must not be credited")
+ def test_skip(self): pass
+if __name__ == "__main__": unittest.main(verbosity=2)
+`
+ if err := os.WriteFile(script, []byte(source), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ controls, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_pass"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"test_added", "test_fail", "test_pass", "test_skip"}; !reflect.DeepEqual(controls, want) {
+ t.Fatalf("new controls must be discovered too: got %v, want %v", controls, want)
+ }
+ if _, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_removed"}); err == nil {
+ t.Fatal("a deleted required control must fail discovery")
+ }
+ for _, name := range []string{"test_pass", "test_skip", "test_fail", "test_missing"} {
+ t.Run(name, func(t *testing.T) {
+ _, err := executePythonFixtureControl(ctx, script, "Controls", name)
+ if (err == nil) != (name == "test_pass") {
+ t.Fatalf("execution verdict for %s: %v", name, err)
+ }
+ })
+ }
+}