Skip to content

The agent that grows with you

fix(process): a supervisor found dead on a broken pipe is reaped too

kshitijk4poor547167d0ee0a
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index aff07327e2c..a7b4a3b5ed6 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -947,6 +947,14 @@ def spawn_owned(argv, *, purpose: str, **popen_kwargs) -> subprocess.Popen:
return proc
+def _reap_in_background(proc) -> None:
+ """Reap an exiting supervisor off the caller's thread. A timed ``Popen.wait`` polls with sleeps
+ (~27 ms per release on macOS), and leaving it unreaped keeps a zombie child that psutil walkers of this
+ process's children trip on; an untimed ``wait()`` blocks in waitpid and returns the moment it exits."""
+ # health: allow HX012 -- the thread only waitpid()s a child that is exiting; no profile-scoped state
+ threading.Thread(target=proc.wait, name="owner-death-supervisor-reaper", daemon=True).start()
+
+
def _update_supervised_pgids(verb: str, pgids) -> None:
"""Send ``register``/``unregister`` for ``pgids`` to this process's supervisor. Failures are
swallowed: losing the safety net must never fail a live child."""
@@ -993,6 +1001,7 @@ def _update_supervised_pgids(verb: str, pgids) -> None:
# It exited between poll() and write(). Drop it so the next call respawns and replays
# from ``_supervised_pgids`` (the set, not the pipe, is the record of what to reap).
_supervisor = None
+ _reap_in_background(proc)
return
if not _supervised_pgids:
@@ -1004,8 +1013,4 @@ def _update_supervised_pgids(verb: str, pgids) -> None:
except (BrokenPipeError, ValueError, OSError):
pass
_supervisor = None
- # Reap it off the caller's thread: a timed Popen.wait polls with sleeps (~25 ms per
- # release), and leaving it unreaped keeps a zombie child that psutil walkers trip on. An
- # untimed wait() blocks in waitpid and returns the moment it exits.
- # health: allow HX012 -- the thread only waitpid()s a child that is exiting; no profile-scoped state
- threading.Thread(target=proc.wait, name="owner-death-supervisor-reaper", daemon=True).start()
+ _reap_in_background(proc)
diff --git a/tests/hermes_cli/test_process_identity_supervisor.py b/tests/hermes_cli/test_process_identity_supervisor.py
index 15c45bb3978..ed0e3875a0c 100644
--- a/tests/hermes_cli/test_process_identity_supervisor.py
+++ b/tests/hermes_cli/test_process_identity_supervisor.py
@@ -13,6 +13,7 @@ import os
import signal
import subprocess
import sys
+import threading
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
@@ -357,6 +358,7 @@ class _FakeSupervisor:
return 1 if self._exited else None
def wait(self, timeout=None):
+ self.waited_on = threading.current_thread()
self.waited = True
return 0
@@ -446,6 +448,7 @@ def test_supervisor_is_released_once_nothing_is_left_to_reap(monkeypatch, all_gr
assert spawned[0].closed, "supervisor kept resident with nothing left to reap"
wait_until(lambda: getattr(spawned[0], "waited", False), timeout=10,
what="the released supervisor to be reaped (a zombie until the next Popen otherwise)")
+ assert spawned[0].waited_on is not threading.current_thread(), "the release blocked on the supervisor's exit"
assert spawned[0].lines()[-1] == "unregister 222", "release happened before the last unregister was sent"
assert process_identity._supervisor is None
@@ -531,13 +534,15 @@ def test_a_broken_pipe_never_propagates_into_a_live_child(monkeypatch, all_group
self.stdin = _Stdin()
- monkeypatch.setattr(process_identity, "_spawn_supervisor", _BrokenPipe)
+ spawned = []
+ monkeypatch.setattr(process_identity, "_spawn_supervisor", lambda: spawned.append(_BrokenPipe()) or spawned[-1])
process_identity.own_process_groups([111]) # must not raise
# Dropped, so the next registration respawns instead of writing into a
- # pipe that is known to be dead.
+ # pipe that is known to be dead; and reaped, so it is not left a zombie.
assert process_identity._supervisor is None
+ wait_until(lambda: getattr(spawned[0], "waited", False), timeout=10, what="the dead supervisor to be reaped")
def test_unregister_after_a_broken_pipe_rebuilds_coverage_for_survivors(monkeypatch, all_groups_alive):