Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Bound passive resident-memory sampling without inventing readings
pulse-triage[bot]f8a52fcaf53b
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.mdindex df28fe2b71..3f8e67ef8d 100644--- a/docs/TROUBLESHOOTING.md+++ b/docs/TROUBLESHOOTING.md@@ -415,32 +415,50 @@ authorised to read these counters; a failed read is unavailable, not zero.# systemd / Proxmox LXC: Pulse resident-memory sample(set -eu- date -u +'%Y-%m-%dT%H:%M:%SZ'- pid=$(systemctl show pulse --property=MainPID --value)- case "$pid" in- ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;;- esac- identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)- if [ -z "$identity" ]; then- printf 'Process identity unavailable.\n' >&2; exit 1- fi- counters=$(awk '- $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" ||- $1 == "RssShmem:" || $1 == "VmSwap:" {- if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1- print; fields++- }- END { if (fields != 5) exit 1 }- ' "/proc/$pid/status")- current_pid=$(systemctl show pulse --property=MainPID --value)- current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)- if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then- printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1+ if timeout --kill-after=1s 8s bash <<'PULSE_MEMORY_SAMPLE'+set -eu+date -u +'%Y-%m-%dT%H:%M:%SZ'+pid=$(systemctl show pulse --property=MainPID --value)+case "$pid" in+ ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;;+esac+identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)+if [ -z "$identity" ]; then+ printf 'Process identity unavailable.\n' >&2; exit 1+fi+counters=$(awk '+ $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" ||+ $1 == "RssShmem:" || $1 == "VmSwap:" {+ if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1+ print; fields+++ }+ END { if (fields != 5) exit 1 }+' "/proc/$pid/status")+current_pid=$(systemctl show pulse --property=MainPID --value)+current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)+if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then+ printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1+fi+printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters"+PULSE_MEMORY_SAMPLE+ then+ :+ else+ status=$?+ printf 'Resident-memory sample unavailable; retain the error and stop sampling.\n' >&2+ exit "$status"fi- printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters")```+The recipe requires GNU `timeout`: it gives the read-only sampling commands+eight seconds, with a one-second termination grace, rather than leaving a hung+reader running indefinitely. It does not stop or restart Pulse. A missing+utility, timeout or reader error means the sample is unavailable; keep the+failure, and do not remove the deadline or rerun it against an unresponsive+installation. No complete process/counter sample is printed before every check+succeeds; the initial UTC timestamp alone is not a memory result.+Linux labels these values `kB`, meaning 1,024 bytes; divide by 1,024 for MiB.`VmRSS` is resident process memory, split into anonymous (`RssAnon`),file-backed (`RssFile`) and shared-memory (`RssShmem`) pages. `VmSwap` is swappeddiff --git a/frontend-modern/public/docs/TROUBLESHOOTING.md b/frontend-modern/public/docs/TROUBLESHOOTING.mdindex df28fe2b71..3f8e67ef8d 100644--- a/frontend-modern/public/docs/TROUBLESHOOTING.md+++ b/frontend-modern/public/docs/TROUBLESHOOTING.md@@ -415,32 +415,50 @@ authorised to read these counters; a failed read is unavailable, not zero.# systemd / Proxmox LXC: Pulse resident-memory sample(set -eu- date -u +'%Y-%m-%dT%H:%M:%SZ'- pid=$(systemctl show pulse --property=MainPID --value)- case "$pid" in- ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;;- esac- identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)- if [ -z "$identity" ]; then- printf 'Process identity unavailable.\n' >&2; exit 1- fi- counters=$(awk '- $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" ||- $1 == "RssShmem:" || $1 == "VmSwap:" {- if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1- print; fields++- }- END { if (fields != 5) exit 1 }- ' "/proc/$pid/status")- current_pid=$(systemctl show pulse --property=MainPID --value)- current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)- if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then- printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1+ if timeout --kill-after=1s 8s bash <<'PULSE_MEMORY_SAMPLE'+set -eu+date -u +'%Y-%m-%dT%H:%M:%SZ'+pid=$(systemctl show pulse --property=MainPID --value)+case "$pid" in+ ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;;+esac+identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)+if [ -z "$identity" ]; then+ printf 'Process identity unavailable.\n' >&2; exit 1+fi+counters=$(awk '+ $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" ||+ $1 == "RssShmem:" || $1 == "VmSwap:" {+ if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1+ print; fields+++ }+ END { if (fields != 5) exit 1 }+' "/proc/$pid/status")+current_pid=$(systemctl show pulse --property=MainPID --value)+current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=)+if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then+ printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1+fi+printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters"+PULSE_MEMORY_SAMPLE+ then+ :+ else+ status=$?+ printf 'Resident-memory sample unavailable; retain the error and stop sampling.\n' >&2+ exit "$status"fi- printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters")```+The recipe requires GNU `timeout`: it gives the read-only sampling commands+eight seconds, with a one-second termination grace, rather than leaving a hung+reader running indefinitely. It does not stop or restart Pulse. A missing+utility, timeout or reader error means the sample is unavailable; keep the+failure, and do not remove the deadline or rerun it against an unresponsive+installation. No complete process/counter sample is printed before every check+succeeds; the initial UTC timestamp alone is not a memory result.+Linux labels these values `kB`, meaning 1,024 bytes; divide by 1,024 for MiB.`VmRSS` is resident process memory, split into anonymous (`RssAnon`),file-backed (`RssFile`) and shared-memory (`RssShmem`) pages. `VmSwap` is swappeddiff --git a/scripts/tests/test_memory_troubleshooting_docs.py b/scripts/tests/test_memory_troubleshooting_docs.pyindex fd9e76c4ae..1728f7d4df 100644--- a/scripts/tests/test_memory_troubleshooting_docs.py+++ b/scripts/tests/test_memory_troubleshooting_docs.py@@ -13,9 +13,11 @@ import osfrom pathlib import Pathimport reimport shutil+import signalimport subprocessimport sysimport tempfile+import timeimport unittest@@ -39,7 +41,7 @@ def recipe() -> str:READER = '''#!/usr/bin/env python3-import json, os, pathlib, subprocess, sys+import json, os, pathlib, subprocess, sys, timename = pathlib.Path(sys.argv[0]).nameargs = sys.argv[1:]root = pathlib.Path(os.environ['RECIPE_FIXTURE'])@@ -48,6 +50,8 @@ previous = [json.loads(line) for line in log.read_text().splitlines()] if log.excount = sum(call['name'] == name for call in previous)with log.open('a') as stream:stream.write(json.dumps({'name': name, 'args': args}) + '\\n')+if name == os.environ.get('HANG_TOOL'):+ time.sleep(30)if name == 'date':assert args == ['-u', '+%Y-%m-%dT%H:%M:%SZ']print('2026-10-06T00:00:00Z')@@ -69,6 +73,25 @@ else:'''+def run_owned_recipe(copied: str, env: dict[str, str]) -> subprocess.CompletedProcess:+ """Keep the existing ten-second assertion; clean up our shell on every exit."""+ with subprocess.Popen(["bash", "-c", copied], env=env, text=True,+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ start_new_session=True) as process:+ try:+ stdout, stderr = process.communicate(timeout=10)+ except subprocess.TimeoutExpired:+ # Never target a service or another test's group. TimeoutExpired+ # remains an error, not a passing or empty sample.+ try:+ os.killpg(process.pid, signal.SIGKILL)+ except ProcessLookupError:+ pass+ process.communicate(timeout=5)+ raise+ return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr)++class MemoryTroubleshootingDocsTest(unittest.TestCase):def exercise(self, *, status: str = COUNTERS, copied: str | None = None,**settings: str):@@ -81,9 +104,9 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):(directory / "status").write_text(status, encoding="utf-8")env = dict(os.environ, PATH=f"{directory}:{os.environ['PATH']}",RECIPE_FIXTURE=str(directory), REAL_AWK=shutil.which("awk"), **settings)- result = subprocess.run(["bash", "-c", copied if copied is not None else recipe()],- env=env, text=True, capture_output=True, timeout=10)- calls = [json.loads(line) for line in (directory / "calls.jsonl").read_text().splitlines()]+ result = run_owned_recipe(copied if copied is not None else recipe(), env)+ log = directory / "calls.jsonl"+ calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []return result, callsdef assert_no_memory_claim(self, result):@@ -167,6 +190,34 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):self.assertEqual(result.returncode, 0, result.stderr)self.assertEqual(result.stdout.count("0 kB"), 5)+ def test_hung_service_reader_is_unavailable_before_the_test_deadline(self):+ started = time.monotonic()+ result, calls = self.exercise(HANG_TOOL="systemctl")+ self.assertLess(time.monotonic() - started, 10)+ self.assertEqual(result.returncode, 124, result.stderr)+ self.assert_no_memory_claim(result)+ self.assertIn("sample unavailable", result.stderr)+ self.assertEqual([call["name"] for call in calls], ["date", "systemctl"])++ def test_hung_counter_reader_never_publishes_identity_or_partial_memory(self):+ started = time.monotonic()+ result, calls = self.exercise(HANG_TOOL="awk")+ self.assertLess(time.monotonic() - started, 10)+ self.assertEqual(result.returncode, 124, result.stderr)+ self.assert_no_memory_claim(result)+ self.assertIn("sample unavailable", result.stderr)+ self.assertEqual([call["name"] for call in calls], ["date", "systemctl", "ps", "awk"])++ def test_missing_deadline_utility_is_not_a_sample_or_an_unbounded_fallback(self):+ copied = recipe().replace("timeout --kill-after=1s 8s bash",+ "pulse-missing-memory-deadline --kill-after=1s 8s bash")+ self.assertNotEqual(copied, recipe())+ result, calls = self.exercise(copied=copied)+ self.assertEqual(result.returncode, 127, result.stderr)+ self.assert_no_memory_claim(result)+ self.assertIn("sample unavailable", result.stderr)+ self.assertEqual(calls, [])+@unittest.skipUnless(sys.platform.startswith("linux"), "Linux /proc recipe")def test_real_linux_status_of_an_owned_disposable_process(self):with tempfile.TemporaryDirectory() as temporary:@@ -178,8 +229,7 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):command.chmod(0o700)env = dict(os.environ, PATH=f"{directory}:{os.environ['PATH']}",RECIPE_CHILD_PID=str(child.pid))- result = subprocess.run(["bash", "-c", recipe()], env=env,- text=True, capture_output=True, timeout=10)+ result = run_owned_recipe(recipe(), env)self.assertEqual(result.returncode, 0, result.stderr)self.assertIn("Pulse process (PID and UTC start)", result.stdout)for field in ("VmRSS", "RssAnon", "RssFile", "RssShmem", "VmSwap"):@@ -197,11 +247,13 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):"1,024 bytes", "approximate and not an atomic snapshot","not specifically the Go heap", "different run","do not measure Pulse RSS", "do not assume its PID 1 is Pulse",+ "requires GNU `timeout`", "initial UTC timestamp alone",+ "do not remove the deadline", "does not stop or restart Pulse","Do not restart Pulse, drop caches, force garbage collection","Do not post a heap dump", "existing screenshots remain useful"):self.assertIn(phrase, guide)self.assertNotRegex(recipe(),- r"\b(?:sudo|restart|kill|rm|truncate|sqlite3|curl|printenv|cat)\b"+ r"(?<!-)\b(?:sudo|restart|kill|rm|truncate|sqlite3|curl|printenv|cat)\b"r"|--follow|--token|cmdline|environ|smaps")