Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Make backend queue controls causal and close child cleanup races
pulse-triage[bot]01f017f3b301
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex bf910ff33d..654fa96728 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -5012,7 +5012,10 @@ exceeds the original vCPU budget; API roots, race/vet checks, 45-minute API andSource `go list` must complete before any runtime child starts. Existing packageweights only order the complete non-API inventory, longest first. Every packagegets a separate data directory, and any failed child fails the gate and cleans-up its siblings. `test_release_backend_queue.py`, discovered by script smoke,+up its siblings. Each task has its own process group: failure signals every+group before waiting, with bounded TERM-to-KILL escalation, rather than racing+a recursive process-tree snapshot against child creation.+`test_release_backend_queue.py`, discovered by script smoke,executes the actual runner with controlled children at one, four and eight CPUs,including partial discovery and both branch failures. The hosted artifactretains source-listed package/API coverage and actual start/finish CPU leases;diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex 4936643fe5..50d08b28ad 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -4869,6 +4869,8 @@ func TestReleaseBackendRaceGateUsesCompleteWorkerPartition(t *testing.T) {`record_task "$!" "$width" "internal/api:$shard_index"`,`free_procs=$((free_procs + task_widths[$completed_pid]))`,`wait -n -p completed_pid`,+ `kill -TERM -- "-$pid"`,+ `kill -KILL -- "-$pid"`,} {if !strings.Contains(backendScript, needle) {t.Fatalf("release backend partition missing coverage contract: %s", needle)diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.shindex 90fee9d49b..6cf85fe0cd 100755--- a/scripts/run-release-backend-tests.sh+++ b/scripts/run-release-backend-tests.sh@@ -90,7 +90,7 @@ if [[ ! "$API_SHARD_TIMEOUT" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; thenexit 2fi-for command_name in go python3 getconf awk pgrep; do+for command_name in go python3 getconf awk; doif ! command -v "$command_name" >/dev/null 2>&1; thenecho "Error: required backend-test command is missing: $command_name" >&2exit 3@@ -325,24 +325,20 @@ run_api_shard() {echo "Completed internal/api shard $shard_index in $((SECONDS - shard_started_seconds))s."}-terminate_tree() {- local parent_pid="$1"- local child_pid- while IFS= read -r child_pid; do- if [ -n "$child_pid" ]; then- terminate_tree "$child_pid"- fi- done < <(pgrep -P "$parent_pid" || true)- kill -TERM "$parent_pid" >/dev/null 2>&1 || true-}-+# Give every asynchronous task its own process group. A recursive pgrep/kill+# snapshot can miss a child forked between enumeration and parent termination.+# Group signalling covers that race, including a leader that already exited.+set -mpids=()declare -A task_widths=() task_names=() task_starts=()free_procs="$VCPUS"stop_tasks() {local pid- for pid in "${pids[@]}"; do terminate_tree "$pid"; done- if [ "${#pids[@]}" -gt 0 ]; then wait "${pids[@]}" >/dev/null 2>&1 || true; fi+ if [ "${#pids[@]}" -eq 0 ]; then return; fi+ for pid in "${pids[@]}"; do kill -TERM -- "-$pid" >/dev/null 2>&1 || true; done+ sleep 1+ for pid in "${pids[@]}"; do kill -KILL -- "-$pid" >/dev/null 2>&1 || true; done+ wait "${pids[@]}" >/dev/null 2>&1 || true}trap stop_tasks EXITtrap 'exit 130' INTdiff --git a/scripts/tests/test_release_backend_queue.py b/scripts/tests/test_release_backend_queue.pyindex 44ed5a7626..135a3f15c9 100644--- a/scripts/tests/test_release_backend_queue.py+++ b/scripts/tests/test_release_backend_queue.py@@ -20,7 +20,7 @@ PACKAGES = [MODULE + name for name in (# All subprocess work is synthetic. The actual shell runner owns discovery,# API partitioning, CPU leases, command flags, data paths and failure cleanup.CHILD = r'''#!/usr/bin/env python3-import fcntl, json, os, re, sys, time+import fcntl, json, os, re, signal, sys, timefrom pathlib import Pathroot = Path(os.environ["QUEUE_FIXTURE"])@@ -56,13 +56,31 @@ else:# The first ordinary package remains busy after a short API shard ends.delay = .7 if name.endswith("scripts/installtests") else .25assert os.environ["CI"] == os.environ["GITHUB_ACTIONS"] == "true"+ignores_term = bool(os.environ.get("QUEUE_IGNORE_TERM")) and name.endswith("scripts/installtests")+if ignores_term:+ signal.signal(signal.SIGTERM, signal.SIG_IGN)+ delay = 10def record(event):with (root / "children.jsonl").open("a") as f:fcntl.flock(f, fcntl.LOCK_EX)f.write(json.dumps(dict(event=event, name=name, pid=os.getpid(),width=int(os.environ["GOMAXPROCS"]), selected=selected,- data=os.environ["PULSE_DATA_DIR"], time=time.monotonic())) + "\n")+ data=os.environ["PULSE_DATA_DIR"], ignores_term=ignores_term,+ time=time.monotonic())) + "\n")record("start")+# A disposable VM may schedule the tiny fixture processes far apart. Use a+# causal barrier, not a .6-second timing race, to prove that a short API shard+# admits another package while both the prefix and first package remain live.+if os.environ["QUEUE_CPUS"] == "4":+ marker = root / "reused-slot"+ if not selected and not name.endswith("scripts/installtests"):+ marker.touch()+ elif name.startswith("api:TestPrefix") or name.endswith("scripts/installtests"):+ deadline = time.monotonic() + 10+ while not marker.exists():+ if time.monotonic() >= deadline:+ sys.exit(97)+ time.sleep(.01)time.sleep(delay)record("finish")if os.environ.get("QUEUE_FAIL") in (name, "api" if selected else "other"):@@ -96,7 +114,8 @@ class ReleaseBackendQueueTest(unittest.TestCase):"--data-root", str(root / "data"), "--api-shards", str(min(3, cpus))],cwd=root, capture_output=True, text=True, timeout=20,env={**os.environ, "PATH": str(root / "bin") + ":" + os.environ["PATH"],- "QUEUE_FIXTURE": str(root), "PULSE_BACKEND_TEST_SHARD_BOUNDARIES": "",+ "QUEUE_FIXTURE": str(root), "QUEUE_CPUS": str(cpus),+ "PULSE_BACKEND_TEST_SHARD_BOUNDARIES": "","PULSE_BACKEND_TEST_SHARD_WEIGHTS": "", **extra})events = root / "children.jsonl"events = [json.loads(line) for line in events.read_text().splitlines()] if events.exists() else []@@ -162,6 +181,14 @@ class ReleaseBackendQueueTest(unittest.TestCase):self.assertTrue(any(row["exit"] == "19" for row in receipts))self.assertNotIn("Race-enabled backend release tests passed", result.stdout)+ def test_failure_escalates_for_a_child_ignoring_term(self):+ result, events, _, _ = self.run_queue(+ QUEUE_FAIL=MODULE + "internal/monitoring", QUEUE_IGNORE_TERM="1")+ self.assertEqual(result.returncode, 19, result.stdout + result.stderr)+ resistant = [e for e in events if e["ignores_term"]]+ self.assertEqual([e["event"] for e in resistant], ["start"])+ self.assertNotIn("Race-enabled backend release tests passed", result.stdout)+if __name__ == "__main__":unittest.main()