Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve completed backend child statuses when draining the queue
pulse-triage[bot]0d2059097a0c
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 654fa96728..65658f9997 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -5015,6 +5015,9 @@ gets a separate data directory, and any failed child fails the gate and cleansup its siblings. Each task has its own process group: failure signals everygroup before waiting, with bounded TERM-to-KILL escalation, rather than racinga recursive process-tree snapshot against child creation.+Already completed children are joined by their retained PID, because Bash+`wait -n` can otherwise omit their saved exit status. A no-job race triggers+reconciliation of the same PIDs, never a task replay or an assumed pass.`test_release_backend_queue.py`, discovered by script smoke,executes the actual runner with controlled children at one, four and eight CPUs,including partial discovery and both branch failures. The hosted artifactdiff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex 50d08b28ad..e8f77b5bfc 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -4869,6 +4869,8 @@ func TestReleaseBackendRaceGateUsesCompleteWorkerPartition(t *testing.T) {`record_task "$!" "$width" "internal/api:$shard_index"`,`free_procs=$((free_procs + task_widths[$completed_pid]))`,`wait -n -p completed_pid`,+ `running="$(jobs -pr)"`,+ `wait "$pid"`,`kill -TERM -- "-$pid"`,`kill -KILL -- "-$pid"`,} {diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.shindex 6cf85fe0cd..cf1bd67bbe 100755--- a/scripts/run-release-backend-tests.sh+++ b/scripts/run-release-backend-tests.sh@@ -358,6 +358,24 @@ record_task() {printf 'start\t%s\t%s\t%s\t-\t%s\n' "$pid" "$width" "$SECONDS" "$name" >> "$RUN_ROOT/coverage/execution.tsv"}+wait_for_task() {+ local pid running+ completed_pid=+ # Bash wait -n ignores children that already finished before the call,+ # whereas wait PID retains their actual status. Drain those first: several+ # fast packages can finish together while the parent schedules replacements.+ running="$(jobs -pr)"+ running=" ${running//$'\n'/ } "+ for pid in "${pids[@]}"; do+ if [[ "$running" != *" $pid "* ]]; then+ completed_pid="$pid"+ wait "$pid"+ return "$?"+ fi+ done+ wait -n -p completed_pid "${pids[@]}"+}+# Beta.3 job 123003682684 finished all API shards in 17 minutes, but kept# the other 169 packages on one slot until the 70-minute job was cancelled.# A completed shard now releases its actual CPU width to whole packages.@@ -379,7 +397,10 @@ while [ "$next_package" -lt "${#OTHER_PACKAGES[@]}" ] || [ "${#pids[@]}" -gt 0 ]next_package=$((next_package + 1))donestatus=0- wait -n -p completed_pid "${pids[@]}" || status=$?+ wait_for_task || status=$?+ # A child can finish between jobs -pr and wait -n. Re-scan the same retained+ # PIDs, never rerun a task or substitute success for an unobserved status.+ if [ -z "${completed_pid:-}" ] && [ "$status" -eq 127 ]; then continue; fiif [ -z "${completed_pid:-}" ]; thenecho "Error: backend child completion was not observed (exit $status)." >&2exit 4diff --git a/scripts/tests/test_release_backend_queue.py b/scripts/tests/test_release_backend_queue.pyindex 135a3f15c9..0c97c6aef9 100644--- a/scripts/tests/test_release_backend_queue.py+++ b/scripts/tests/test_release_backend_queue.py@@ -5,6 +5,7 @@ import csvimport jsonimport osfrom pathlib import Path+import reimport shutilimport subprocessimport tempfile@@ -189,6 +190,27 @@ class ReleaseBackendQueueTest(unittest.TestCase):self.assertEqual([e["event"] for e in resistant], ["start"])self.assertNotIn("Race-enabled backend release tests passed", result.stdout)+ def test_wait_preserves_already_completed_child_status(self):+ source = (ROOT / "scripts/run-release-backend-tests.sh").read_text()+ helper = re.search(r"^wait_for_task\(\) \{\n.*?^\}", source, re.M | re.S)+ self.assertIsNotNone(helper)+ for child_exit in (0, 19, 127):+ with self.subTest(child_exit=child_exit):+ script = helper.group() + f'''+set -m+bash -c 'exit {child_exit}' &+pids=("$!")+# Establish that the child is no longer running before joining it. A saved+# failure, even exit127, must not be confused with wait -n's no-job result.+while kill -0 "${{pids[0]}}" 2>/dev/null; do sleep .01; done+status=0+wait_for_task || status=$?+test "$completed_pid" = "${{pids[0]}}" && test "$status" = {child_exit}+'''+ result = subprocess.run(["bash", "-c", script], capture_output=True,+ text=True, timeout=5)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+if __name__ == "__main__":unittest.main()