Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(minikube): r1-l1 reap a real zombie-only group without injection

Alfredo Lopez72b3ec315a2d
diff --git a/scripts/minikube/run-with-deadline.mjs b/scripts/minikube/run-with-deadline.mjs
index 99b3dcecd..679c6fcde 100755
--- a/scripts/minikube/run-with-deadline.mjs
+++ b/scripts/minikube/run-with-deadline.mjs
@@ -133,6 +133,10 @@ function killProcessGroup(signal) {
return null;
}
+// EPERM counts as alive. macOS answers signal 0 for a group that holds only
+// zombies with EPERM too, the same as SIGKILL, so an exiting group reads as
+// alive until its last zombie is reaped and the probe turns ESRCH; the reap
+// window in reapProcessGroup must span that reap.
function processGroupExists() {
if (!child.pid) return false;
if (process.platform === "win32") {
diff --git a/scripts/tests/fixtures/run-with-deadline/deny-group-sigkill.mjs b/scripts/tests/fixtures/run-with-deadline/deny-group-sigkill.mjs
index b8386d0a7..54b6bbae0 100644
--- a/scripts/tests/fixtures/run-with-deadline/deny-group-sigkill.mjs
+++ b/scripts/tests/fixtures/run-with-deadline/deny-group-sigkill.mjs
@@ -5,8 +5,9 @@
// after the wrapped command has exited. That race is not reproducible on
// demand, so this preload makes every signal in DENY_GROUP_SIGNALS (default
// SIGKILL) addressed to a process group fail the same way. Signal 0 still
-// reaches the real kill(), so the runner's liveness probe reports whether the
-// group is actually gone.
+// reaches the real kill(), so the runner's liveness probe reports the group's
+// real state; zombie-group-member.py covers the zombie-only state, where the
+// real probe answers EPERM on macOS.
//
// Each denial is logged with the group's real state just before it
// (before=alive|ESRCH|EPERM from a real kill(pid, 0)), so a test can tell
diff --git a/scripts/tests/fixtures/run-with-deadline/zombie-group-member.py b/scripts/tests/fixtures/run-with-deadline/zombie-group-member.py
new file mode 100644
index 000000000..67a731e1b
--- /dev/null
+++ b/scripts/tests/fixtures/run-with-deadline/zombie-group-member.py
@@ -0,0 +1,59 @@
+# Run in the background by the wrapped command of test-run-with-deadline-reap.sh.
+#
+# Leaves the wrapped command's process group holding only an unreaped zombie,
+# with the real kernel and no injected errors: this process forks a member,
+# moves itself to a new session (so it is no longer in the group), and keeps
+# the member unreaped until the group leader has exited and been reaped by the
+# runner. It then records what kill(-pgid, 0), the runner's liveness probe,
+# answers in that state (macOS: EPERM, Linux: ok), holds the state across
+# several of the runner's 25 ms polls, and reaps the member.
+#
+# Log lines: `zombie member=<pid>`, `probe=<ok|errno name>`, `reaped member=<pid>`.
+import errno
+import os
+import sys
+import time
+
+log = sys.argv[1]
+
+
+def record(line):
+ with open(log, "a", encoding="utf-8") as handle:
+ handle.write(line + "\n")
+
+
+leader = os.getpgid(0)
+member = os.fork()
+if member == 0:
+ os._exit(0)
+
+os.setsid()
+# Wait for the member to exit without reaping it.
+os.waitid(os.P_PID, member, os.WEXITED | os.WNOWAIT)
+record(f"zombie member={member}")
+
+deadline = time.monotonic() + 5
+while True:
+ try:
+ os.kill(leader, 0)
+ except ProcessLookupError:
+ break
+ except PermissionError:
+ # macOS answers EPERM for the leader while it is a zombie the runner
+ # has not reaped yet; it is still there.
+ pass
+ if time.monotonic() >= deadline:
+ record(f"leader-not-reaped leader={leader}")
+ sys.exit(1)
+ time.sleep(0.01)
+
+try:
+ os.kill(-leader, 0)
+ record("probe=ok")
+except OSError as error:
+ assert error.errno is not None, error
+ record(f"probe={errno.errorcode[error.errno]}")
+
+time.sleep(0.2)
+os.waitpid(member, 0)
+record(f"reaped member={member}")
diff --git a/scripts/tests/test-run-with-deadline-reap.sh b/scripts/tests/test-run-with-deadline-reap.sh
index 26d123a77..8aec200a5 100755
--- a/scripts/tests/test-run-with-deadline-reap.sh
+++ b/scripts/tests/test-run-with-deadline-reap.sh
@@ -10,6 +10,7 @@ set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
RUNNER="$ROOT/scripts/minikube/run-with-deadline.mjs"
PRELOAD="$ROOT/scripts/tests/fixtures/run-with-deadline/deny-group-sigkill.mjs"
+ZOMBIE_HELPER="$ROOT/scripts/tests/fixtures/run-with-deadline/zombie-group-member.py"
TMP_DIR="$(mktemp -d)"
LEFTOVER_PIDS=()
# Invoked indirectly by the EXIT trap.
@@ -172,6 +173,62 @@ assert_denied_reap_of_a_live_group_fails_loud() {
fi
}
+assert_a_real_zombie_only_group_keeps_the_child_status() {
+ local output="$TMP_DIR/zombie.out" zombie_log="$TMP_DIR/zombie.log" status=0
+ : >"$zombie_log"
+ # No preload: the kernel answers both the reap and the liveness probe. The
+ # wrapped command exits only once the helper holds an unreaped zombie in the
+ # group, so the reap runs while the group holds nothing else.
+ # shellcheck disable=SC2016
+ node "$RUNNER" --timeout-seconds 10 --heartbeat-seconds 1 --kill-grace-seconds 1 \
+ --label zombie-exit -- bash -c '
+ python3 "$1" "$2" &
+ for _ in $(seq 250); do
+ grep -Fq "zombie member=" "$2" && exit 7
+ sleep 0.02
+ done
+ exit 99' _ "$ZOMBIE_HELPER" "$zombie_log" >"$output" 2>&1 || status=$?
+
+ # The helper outlives the wrapped command; wait for its last line so the
+ # probe it recorded is the one taken inside the runner's reap window.
+ local _
+ for _ in $(seq 250); do
+ grep -Fq 'reaped member=' "$zombie_log" && break
+ sleep 0.02
+ done
+
+ local expected_probe reap_event_ok=false
+ case "$(uname -s)" in
+ Darwin)
+ # macOS refuses both SIGKILL and signal 0 for a zombie-only group, so the
+ # runner sees the group alive until the zombie is reaped.
+ expected_probe=EPERM
+ grep -Fq 'event=reap-permission-denied groupGone=true' "$output" \
+ && reap_event_ok=true
+ ;;
+ Linux)
+ # Linux signals a zombie successfully, so the reap is never refused.
+ expected_probe=ok
+ grep -Fq 'event=reap-' "$output" || reap_event_ok=true
+ ;;
+ *)
+ fail "zombie-only group: unsupported platform $(uname -s)"
+ return
+ ;;
+ esac
+
+ if [[ "$status" -eq 7 && "$reap_event_ok" == true ]] \
+ && grep -Fq 'zombie member=' "$zombie_log" \
+ && grep -Fxq "probe=$expected_probe" "$zombie_log" \
+ && grep -Fq 'reaped member=' "$zombie_log" \
+ && grep -Fq 'event=exit' "$output" && grep -Fq 'exitCode=7' "$output" \
+ && ! grep -Fq 'event=reap-failed' "$output"; then
+ pass "a real zombie-only group (probe=$expected_probe) keeps the child's exit status 7"
+ else
+ fail "zombie-only group (status=$status): $(cat "$zombie_log") $(cat "$output")"
+ fi
+}
+
assert_an_undenied_reap_reports_no_permission_event() {
local output="$TMP_DIR/control.out" status=0
node "$RUNNER" --timeout-seconds 10 --heartbeat-seconds 1 --kill-grace-seconds 1 \
@@ -205,6 +262,7 @@ assert_denied_reap_survives_a_wall_clock_step
assert_denied_reap_of_a_group_the_sigterm_emptied_keeps_the_timeout_status
assert_refused_sigterm_is_reported_and_the_reap_still_resolves
assert_denied_reap_of_a_live_group_fails_loud
+assert_a_real_zombie_only_group_keeps_the_child_status
assert_an_undenied_reap_reports_no_permission_event
assert_every_defined_case_is_invoked