Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Make native NAS cleanup proof portable and observable

pulse-triage[bot]81d614e937b4
diff --git a/scripts/installtests/agent_nas_teardown.py b/scripts/installtests/agent_nas_teardown.py
index 7f7ffcc7ae..53bca0b3c7 100644
--- a/scripts/installtests/agent_nas_teardown.py
+++ b/scripts/installtests/agent_nas_teardown.py
@@ -11,8 +11,11 @@ import shlex
import shutil
import stat
import subprocess
+import sys
import tempfile
+import time
import unittest
+from unittest import mock
ROOT = Path(__file__).resolve().parents[2]
SOURCE = Path(os.environ.get("PULSE_AGENT_NAS_SOURCE", ROOT / "scripts/install.sh"))
@@ -80,11 +83,11 @@ exec ''' + shlex.quote(str(executable)) + ' -ww "$@"\n')
return target
-MODEL = r'''#!/usr/bin/python3
+MODEL = r'''
import json, os, pathlib, sys
root = pathlib.Path(os.environ['FIXTURE'])
path = root / 'model.json'
-v = json.loads(path.read_text()); role, args = pathlib.Path(sys.argv[0]).name, sys.argv[1:]
+v = json.loads(path.read_text()); role, args = sys.argv[1], sys.argv[2:]
fault = v['fault']
def event(s):
with (root / 'events').open('a') as f: f.write(s + '\n')
@@ -135,6 +138,65 @@ sys.exit(0)
'''
+def write_model_tools(root, tools):
+ # Use the interpreter selected by the native runner, not Darwin's unrelated
+ # /usr/bin/python3. These models use only the standard library: repeated
+ # site startup is unnecessary, and site hooks must not alter fault controls.
+ model = root / 'model.py'
+ model.write_text(MODEL)
+ for name in ('ps', 'kill', 'flash'):
+ tool = tools / name
+ tool.write_text('#!/bin/sh\nexec ' + shlex.quote(sys.executable)
+ + ' -I -S ' + shlex.quote(str(model)) + ' ' + name + ' "$@"\n')
+ tool.chmod(0o700)
+
+
+def observed_process_absence(pid_file):
+ # /proc does not exist on Darwin. Missing process evidence is not proof
+ # of cleanup; only a kernel ESRCH for the recorded positive PID is absence.
+ try:
+ pid = int(pid_file.read_text())
+ except (OSError, ValueError):
+ return False
+ if pid <= 0:
+ return False
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return True
+ except OSError:
+ return False
+ return False
+
+
+class NASTimingResult(unittest.TextTestResult):
+ """Retain progress before the outer deadline, without changing any budget."""
+
+ def startTest(self, test):
+ self.started = time.monotonic()
+ if not hasattr(self, 'suite_started'):
+ self.suite_started = self.started
+ self.subcases = 0
+ self.progress('start', test)
+ super().startTest(test)
+
+ def addSubTest(self, test, subtest, err):
+ self.subcases += 1
+ super().addSubTest(test, subtest, err)
+
+ def stopTest(self, test):
+ super().stopTest(test)
+ self.progress('complete', test)
+
+ def progress(self, phase, test):
+ now = time.monotonic()
+ self.stream.writeln('NAS fixture timing: ' + json.dumps(dict(
+ phase=phase, control=test._testMethodName, completed_subcases=self.subcases,
+ suite_seconds=round(now - self.suite_started, 6),
+ control_seconds=round(now - self.started, 6)), sort_keys=True))
+ self.stream.flush()
+
+
class AgentNASTeardown(unittest.TestCase):
source = SOURCE.read_text()
@@ -143,8 +205,7 @@ class AgentNASTeardown(unittest.TestCase):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
tools = root / "tools"; tools.mkdir()
- for name in ("ps", "kill", "flash"):
- p = tools / name; p.write_text(MODEL); p.chmod(0o700)
+ write_model_tools(root, tools)
for name in ("state", "helper", "profile", "bin", "flash", "legacy", "foreign"):
d = root / name; d.mkdir(mode=0o700)
if name in ("state", "helper", "profile", "foreign"):
@@ -285,7 +346,7 @@ remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }
after = self.snapshot(root)
content = bootfile.read_bytes() if bootfile.is_file() and not bootfile.is_symlink() else None
recovered = {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("retired")}
- timed_stopped = not (root / "timed-pid").exists() or not Path('/proc/' + (root / "timed-pid").read_text()).exists()
+ timed_stopped = observed_process_absence(root / "timed-pid")
return result, second, events, before, after, json.loads((root / "model.json").read_text()), content, original_boot, recovered, timed_stopped
@staticmethod
@@ -410,6 +471,90 @@ remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }
self.assertNotIn('/synthetic/private-only', r[0].stdout); self.assertIn('200 1 other\n', r[0].stdout)
r = self.exercise('QNAP', 'timeout-process'); self.retained(r); self.assertTrue(r[9])
+ def test_private_model_uses_selected_isolated_interpreter(self):
+ with tempfile.TemporaryDirectory(prefix='nas model ') as directory:
+ root = Path(directory)
+ tools = root / 'tools'; tools.mkdir()
+ invocations = root / 'interpreter-arguments'
+ interpreter = root / 'selected python'
+ interpreter.write_text('#!/bin/sh\nprintf "%s\\n" "$@" >> '
+ + shlex.quote(str(invocations)) + '\nexec '
+ + shlex.quote(sys.executable) + ' "$@"\n')
+ interpreter.chmod(0o700)
+ with mock.patch.object(sys, 'executable', str(interpreter)):
+ write_model_tools(root, tools)
+ # A site hook would contaminate model state if startup isolation
+ # disappeared. No external package, service or process is involved.
+ (root / 'sitecustomize.py').write_text(
+ 'from pathlib import Path\nPath(' + repr(str(root / 'site-ran')) + ').touch()\n')
+ (root / 'model.json').write_text(json.dumps(dict(
+ platform='QNAP', fault='', reads=0, watchdog=True, collector=True)))
+ (root / 'flash').mkdir(); (root / 'flash/autorun.sh').write_text('#!/bin/sh\n')
+ env = dict(os.environ, FIXTURE=str(root), PYTHONPATH=str(root))
+ commands = (
+ ('ps', '-ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'comm=', '-o', 'args='),
+ ('kill', '-TERM', '100'),
+ ('flash', 'mount_flash_config'),
+ )
+ for name, *args in commands:
+ result = subprocess.run([str(tools / name), *args], env=env,
+ capture_output=True, text=True, timeout=5)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(result.stderr, '')
+ recorded = invocations.read_text().splitlines()
+ self.assertEqual(recorded.count('-I'), 3)
+ self.assertEqual(recorded.count('-S'), 3)
+ self.assertEqual(recorded.count(str(root / 'model.py')), 3)
+ self.assertFalse((root / 'site-ran').exists())
+ self.assertEqual((root / 'events').read_text().splitlines(), [
+ 'process:read', 'signal:100', 'boot:absent-at-signal', 'flash:mount_flash_config'])
+ self.assertFalse(json.loads((root / 'model.json').read_text())['watchdog'])
+
+ def test_timeout_child_absence_requires_observed_exit(self):
+ with tempfile.TemporaryDirectory() as directory:
+ pid_file = Path(directory) / 'child-pid'
+ with mock.patch.object(os, 'kill') as probe:
+ for value in (None, '', 'invalid', '0', '-1'):
+ if value is not None:
+ pid_file.write_text(value)
+ self.assertFalse(observed_process_absence(pid_file))
+ probe.assert_not_called()
+ process = subprocess.Popen(['sleep', '30'])
+ try:
+ pid_file.write_text(str(process.pid))
+ self.assertFalse(observed_process_absence(pid_file), 'live child reported absent')
+ with mock.patch.object(os, 'kill', side_effect=PermissionError):
+ self.assertFalse(observed_process_absence(pid_file), 'refusal reported absence')
+ finally:
+ process.terminate()
+ process.wait(timeout=3)
+ self.assertTrue(observed_process_absence(pid_file), 'reaped child still reported live')
+
+ def test_control_timing_reports_cumulative_and_per_control_time(self):
+ import io
+
+ class InertControl(unittest.TestCase):
+ def test_inert(self):
+ for _ in range(2):
+ with self.subTest():
+ self.assertTrue(True)
+
+ stream = io.StringIO()
+ result = unittest.TextTestRunner(stream=stream, verbosity=2,
+ resultclass=NASTimingResult).run(
+ unittest.defaultTestLoader.loadTestsFromTestCase(InertControl))
+ self.assertTrue(result.wasSuccessful())
+ records = [json.loads(line.removeprefix('NAS fixture timing: '))
+ for line in stream.getvalue().splitlines() if line.startswith('NAS fixture timing: ')]
+ self.assertEqual([r['phase'] for r in records], ['start', 'complete'])
+ self.assertEqual([r['completed_subcases'] for r in records], [0, 2])
+ for record in records:
+ self.assertEqual(set(record), {'phase', 'control', 'completed_subcases',
+ 'suite_seconds', 'control_seconds'})
+ self.assertEqual(record['control'], 'test_inert')
+ self.assertGreaterEqual(record['suite_seconds'], record['control_seconds'])
+ self.assertGreaterEqual(records[1]['control_seconds'], records[0]['control_seconds'])
+
def test_live_unraid_boot_shell_is_unknown_not_a_signal_target(self):
r = self.exercise('Unraid', 'live-go'); events = self.retained(r)
self.assertEqual(r[6], r[7]); self.assertFalse(any(e.startswith('signal:') for e in events))
@@ -561,4 +706,5 @@ remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }
self.assertIn('[[ "$uninstall_upstart" != true && -z "$uninstall_nas_platform" ]]', caller)
-if __name__ == '__main__': unittest.main(verbosity=2)
+if __name__ == '__main__':
+ unittest.main(verbosity=2, testRunner=unittest.TextTestRunner(verbosity=2, resultclass=NASTimingResult))
diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go
index d1071241f4..0ed61bf65b 100644
--- a/scripts/installtests/install_sh_test.go
+++ b/scripts/installtests/install_sh_test.go
@@ -8576,6 +8576,9 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {
"test_recovered_ordinary_linux_state_is_not_qnap_platform_authority",
"test_live_unraid_boot_shell_is_unknown_not_a_signal_target",
"test_actual_generated_watchdogs_and_children_become_quiescent",
+ "test_private_model_uses_selected_isolated_interpreter",
+ "test_timeout_child_absence_requires_observed_exit",
+ "test_control_timing_reports_cumulative_and_per_control_time",
"test_private_process_reader_requests_native_basename_and_full_argv",
"test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats",
"test_caller_has_no_blanket_nas_or_legacy_tree_deletion",
@@ -8584,7 +8587,7 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {
t.Fatalf("NAS removal control %s did not pass:\n%s", name, output)
}
}
- if !strings.Contains(string(output), "\nRan 20 tests in ") || !strings.Contains(string(output), "\nOK\n") {
+ if !strings.Contains(string(output), "\nRan 23 tests in ") || !strings.Contains(string(output), "\nOK\n") {
t.Fatalf("incomplete NAS removal verdict:\n%s", output)
}
t.Logf("%s", output)