Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve completed backend child statuses when draining the queue

pulse-triage[bot]0d2059097a0c
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 654fa96728..65658f9997 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -5015,6 +5015,9 @@ gets a separate data directory, and any failed child fails the gate and cleans
up its siblings. Each task has its own process group: failure signals every
group before waiting, with bounded TERM-to-KILL escalation, rather than racing
a recursive process-tree snapshot against child creation.
+Already completed children are joined by their retained PID, because Bash
+`wait -n` can otherwise omit their saved exit status. A no-job race triggers
+reconciliation of the same PIDs, never a task replay or an assumed pass.
`test_release_backend_queue.py`, discovered by script smoke,
executes the actual runner with controlled children at one, four and eight CPUs,
including partial discovery and both branch failures. The hosted artifact
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 50d08b28ad..e8f77b5bfc 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -4869,6 +4869,8 @@ func TestReleaseBackendRaceGateUsesCompleteWorkerPartition(t *testing.T) {
`record_task "$!" "$width" "internal/api:$shard_index"`,
`free_procs=$((free_procs + task_widths[$completed_pid]))`,
`wait -n -p completed_pid`,
+ `running="$(jobs -pr)"`,
+ `wait "$pid"`,
`kill -TERM -- "-$pid"`,
`kill -KILL -- "-$pid"`,
} {
diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.sh
index 6cf85fe0cd..cf1bd67bbe 100755
--- a/scripts/run-release-backend-tests.sh
+++ b/scripts/run-release-backend-tests.sh
@@ -358,6 +358,24 @@ record_task() {
printf 'start\t%s\t%s\t%s\t-\t%s\n' "$pid" "$width" "$SECONDS" "$name" >> "$RUN_ROOT/coverage/execution.tsv"
}
+wait_for_task() {
+ local pid running
+ completed_pid=
+ # Bash wait -n ignores children that already finished before the call,
+ # whereas wait PID retains their actual status. Drain those first: several
+ # fast packages can finish together while the parent schedules replacements.
+ running="$(jobs -pr)"
+ running=" ${running//$'\n'/ } "
+ for pid in "${pids[@]}"; do
+ if [[ "$running" != *" $pid "* ]]; then
+ completed_pid="$pid"
+ wait "$pid"
+ return "$?"
+ fi
+ done
+ wait -n -p completed_pid "${pids[@]}"
+}
+
# Beta.3 job 123003682684 finished all API shards in 17 minutes, but kept
# the other 169 packages on one slot until the 70-minute job was cancelled.
# A completed shard now releases its actual CPU width to whole packages.
@@ -379,7 +397,10 @@ while [ "$next_package" -lt "${#OTHER_PACKAGES[@]}" ] || [ "${#pids[@]}" -gt 0 ]
next_package=$((next_package + 1))
done
status=0
- wait -n -p completed_pid "${pids[@]}" || status=$?
+ wait_for_task || status=$?
+ # A child can finish between jobs -pr and wait -n. Re-scan the same retained
+ # PIDs, never rerun a task or substitute success for an unobserved status.
+ if [ -z "${completed_pid:-}" ] && [ "$status" -eq 127 ]; then continue; fi
if [ -z "${completed_pid:-}" ]; then
echo "Error: backend child completion was not observed (exit $status)." >&2
exit 4
diff --git a/scripts/tests/test_release_backend_queue.py b/scripts/tests/test_release_backend_queue.py
index 135a3f15c9..0c97c6aef9 100644
--- a/scripts/tests/test_release_backend_queue.py
+++ b/scripts/tests/test_release_backend_queue.py
@@ -5,6 +5,7 @@ import csv
import json
import os
from pathlib import Path
+import re
import shutil
import subprocess
import tempfile
@@ -189,6 +190,27 @@ class ReleaseBackendQueueTest(unittest.TestCase):
self.assertEqual([e["event"] for e in resistant], ["start"])
self.assertNotIn("Race-enabled backend release tests passed", result.stdout)
+ def test_wait_preserves_already_completed_child_status(self):
+ source = (ROOT / "scripts/run-release-backend-tests.sh").read_text()
+ helper = re.search(r"^wait_for_task\(\) \{\n.*?^\}", source, re.M | re.S)
+ self.assertIsNotNone(helper)
+ for child_exit in (0, 19, 127):
+ with self.subTest(child_exit=child_exit):
+ script = helper.group() + f'''
+set -m
+bash -c 'exit {child_exit}' &
+pids=("$!")
+# Establish that the child is no longer running before joining it. A saved
+# failure, even exit127, must not be confused with wait -n's no-job result.
+while kill -0 "${{pids[0]}}" 2>/dev/null; do sleep .01; done
+status=0
+wait_for_task || status=$?
+test "$completed_pid" = "${{pids[0]}}" && test "$status" = {child_exit}
+'''
+ result = subprocess.run(["bash", "-c", script], capture_output=True,
+ text=True, timeout=5)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
if __name__ == "__main__":
unittest.main()