Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Make backend queue controls causal and close child cleanup races

pulse-triage[bot]01f017f3b301
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index bf910ff33d..654fa96728 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -5012,7 +5012,10 @@ exceeds the original vCPU budget; API roots, race/vet checks, 45-minute API and
Source `go list` must complete before any runtime child starts. Existing package
weights only order the complete non-API inventory, longest first. Every package
gets a separate data directory, and any failed child fails the gate and cleans
-up its siblings. `test_release_backend_queue.py`, discovered by script smoke,
+up its siblings. Each task has its own process group: failure signals every
+group before waiting, with bounded TERM-to-KILL escalation, rather than racing
+a recursive process-tree snapshot against child creation.
+`test_release_backend_queue.py`, discovered by script smoke,
executes the actual runner with controlled children at one, four and eight CPUs,
including partial discovery and both branch failures. The hosted artifact
retains source-listed package/API coverage and actual start/finish CPU leases;
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 4936643fe5..50d08b28ad 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -4869,6 +4869,8 @@ func TestReleaseBackendRaceGateUsesCompleteWorkerPartition(t *testing.T) {
`record_task "$!" "$width" "internal/api:$shard_index"`,
`free_procs=$((free_procs + task_widths[$completed_pid]))`,
`wait -n -p completed_pid`,
+ `kill -TERM -- "-$pid"`,
+ `kill -KILL -- "-$pid"`,
} {
if !strings.Contains(backendScript, needle) {
t.Fatalf("release backend partition missing coverage contract: %s", needle)
diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.sh
index 90fee9d49b..6cf85fe0cd 100755
--- a/scripts/run-release-backend-tests.sh
+++ b/scripts/run-release-backend-tests.sh
@@ -90,7 +90,7 @@ if [[ ! "$API_SHARD_TIMEOUT" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then
exit 2
fi
-for command_name in go python3 getconf awk pgrep; do
+for command_name in go python3 getconf awk; do
if ! command -v "$command_name" >/dev/null 2>&1; then
echo "Error: required backend-test command is missing: $command_name" >&2
exit 3
@@ -325,24 +325,20 @@ run_api_shard() {
echo "Completed internal/api shard $shard_index in $((SECONDS - shard_started_seconds))s."
}
-terminate_tree() {
- local parent_pid="$1"
- local child_pid
- while IFS= read -r child_pid; do
- if [ -n "$child_pid" ]; then
- terminate_tree "$child_pid"
- fi
- done < <(pgrep -P "$parent_pid" || true)
- kill -TERM "$parent_pid" >/dev/null 2>&1 || true
-}
-
+# Give every asynchronous task its own process group. A recursive pgrep/kill
+# snapshot can miss a child forked between enumeration and parent termination.
+# Group signalling covers that race, including a leader that already exited.
+set -m
pids=()
declare -A task_widths=() task_names=() task_starts=()
free_procs="$VCPUS"
stop_tasks() {
local pid
- for pid in "${pids[@]}"; do terminate_tree "$pid"; done
- if [ "${#pids[@]}" -gt 0 ]; then wait "${pids[@]}" >/dev/null 2>&1 || true; fi
+ if [ "${#pids[@]}" -eq 0 ]; then return; fi
+ for pid in "${pids[@]}"; do kill -TERM -- "-$pid" >/dev/null 2>&1 || true; done
+ sleep 1
+ for pid in "${pids[@]}"; do kill -KILL -- "-$pid" >/dev/null 2>&1 || true; done
+ wait "${pids[@]}" >/dev/null 2>&1 || true
}
trap stop_tasks EXIT
trap 'exit 130' INT
diff --git a/scripts/tests/test_release_backend_queue.py b/scripts/tests/test_release_backend_queue.py
index 44ed5a7626..135a3f15c9 100644
--- a/scripts/tests/test_release_backend_queue.py
+++ b/scripts/tests/test_release_backend_queue.py
@@ -20,7 +20,7 @@ PACKAGES = [MODULE + name for name in (
# All subprocess work is synthetic. The actual shell runner owns discovery,
# API partitioning, CPU leases, command flags, data paths and failure cleanup.
CHILD = r'''#!/usr/bin/env python3
-import fcntl, json, os, re, sys, time
+import fcntl, json, os, re, signal, sys, time
from pathlib import Path
root = Path(os.environ["QUEUE_FIXTURE"])
@@ -56,13 +56,31 @@ else:
# The first ordinary package remains busy after a short API shard ends.
delay = .7 if name.endswith("scripts/installtests") else .25
assert os.environ["CI"] == os.environ["GITHUB_ACTIONS"] == "true"
+ignores_term = bool(os.environ.get("QUEUE_IGNORE_TERM")) and name.endswith("scripts/installtests")
+if ignores_term:
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ delay = 10
def record(event):
with (root / "children.jsonl").open("a") as f:
fcntl.flock(f, fcntl.LOCK_EX)
f.write(json.dumps(dict(event=event, name=name, pid=os.getpid(),
width=int(os.environ["GOMAXPROCS"]), selected=selected,
- data=os.environ["PULSE_DATA_DIR"], time=time.monotonic())) + "\n")
+ data=os.environ["PULSE_DATA_DIR"], ignores_term=ignores_term,
+ time=time.monotonic())) + "\n")
record("start")
+# A disposable VM may schedule the tiny fixture processes far apart. Use a
+# causal barrier, not a .6-second timing race, to prove that a short API shard
+# admits another package while both the prefix and first package remain live.
+if os.environ["QUEUE_CPUS"] == "4":
+ marker = root / "reused-slot"
+ if not selected and not name.endswith("scripts/installtests"):
+ marker.touch()
+ elif name.startswith("api:TestPrefix") or name.endswith("scripts/installtests"):
+ deadline = time.monotonic() + 10
+ while not marker.exists():
+ if time.monotonic() >= deadline:
+ sys.exit(97)
+ time.sleep(.01)
time.sleep(delay)
record("finish")
if os.environ.get("QUEUE_FAIL") in (name, "api" if selected else "other"):
@@ -96,7 +114,8 @@ class ReleaseBackendQueueTest(unittest.TestCase):
"--data-root", str(root / "data"), "--api-shards", str(min(3, cpus))],
cwd=root, capture_output=True, text=True, timeout=20,
env={**os.environ, "PATH": str(root / "bin") + ":" + os.environ["PATH"],
- "QUEUE_FIXTURE": str(root), "PULSE_BACKEND_TEST_SHARD_BOUNDARIES": "",
+ "QUEUE_FIXTURE": str(root), "QUEUE_CPUS": str(cpus),
+ "PULSE_BACKEND_TEST_SHARD_BOUNDARIES": "",
"PULSE_BACKEND_TEST_SHARD_WEIGHTS": "", **extra})
events = root / "children.jsonl"
events = [json.loads(line) for line in events.read_text().splitlines()] if events.exists() else []
@@ -162,6 +181,14 @@ class ReleaseBackendQueueTest(unittest.TestCase):
self.assertTrue(any(row["exit"] == "19" for row in receipts))
self.assertNotIn("Race-enabled backend release tests passed", result.stdout)
+ def test_failure_escalates_for_a_child_ignoring_term(self):
+ result, events, _, _ = self.run_queue(
+ QUEUE_FAIL=MODULE + "internal/monitoring", QUEUE_IGNORE_TERM="1")
+ self.assertEqual(result.returncode, 19, result.stdout + result.stderr)
+ resistant = [e for e in events if e["ignores_term"]]
+ self.assertEqual([e["event"] for e in resistant], ["start"])
+ self.assertNotIn("Race-enabled backend release tests passed", result.stdout)
+
if __name__ == "__main__":
unittest.main()