Skip to content

The agent that grows with you

fix(execute_code): a kernel's cell subprocesses die with the Hermes process that ran them (orphans_after_10s 3 -> 0)

kshitijk4pooreb834d16f2bf
diff --git a/tests/tools/test_code_kernel.py b/tests/tools/test_code_kernel.py
index 9846656f0ed..f05e4e861ec 100644
--- a/tests/tools/test_code_kernel.py
+++ b/tests/tools/test_code_kernel.py
@@ -107,8 +107,9 @@ class TestKernelLifecycle(unittest.TestCase):
def test_kernel_exits_when_its_backend_parent_dies(self):
"""A kernel must not outlive the host that spawned it, even when the
host dies without cleanup (SIGKILL/OOM/crash). Windows: inherited
- SYNCHRONIZE handle; POSIX: inherited death pipe. Both are proven the
- same way — kill the host mid-cell, the kernel is gone within seconds."""
+ SYNCHRONIZE handle; POSIX: the kernel's group is registered with the
+ owner-death supervisor. Both are proven the same way — kill the host
+ mid-cell, the kernel is gone within seconds."""
import psutil
repo_root = str(Path(__file__).resolve().parents[2])
diff --git a/tests/tools/test_code_kernel_owned_group.py b/tests/tools/test_code_kernel_owned_group.py
new file mode 100644
index 00000000000..d77b59f10c2
--- /dev/null
+++ b/tests/tools/test_code_kernel_owned_group.py
@@ -0,0 +1,150 @@
+"""POSIX session kernels: the kernel's whole process group dies with its owner, however the owner dies.
+
+The owners are real host processes importing the real ``tools.code_kernel._spawn``; each starts a kernel
+whose cell spawns a subprocess of its own (it stays in the kernel's group, as every cell child does).
+One owner is SIGKILLed so no Python cleanup of ours can run; the other is the live-owner control.
+"""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+import psutil
+import pytest
+
+from tests.waiting import wait_until
+
+pytestmark = pytest.mark.platforms("posix")
+
+_OWNER = r'''
+import json, sys, time
+from pathlib import Path
+root, role = Path(sys.argv[1]), sys.argv[2]
+from tools.code_kernel import SessionKernel, _spawn
+k = SessionKernel((role,))
+_spawn(k, task_id=role, child_python=sys.executable, child_cwd="", sandbox_tools=frozenset(),
+ max_tool_calls=1)
+receipt = str(root / (role + ".json"))
+code = (
+ "import json, os, subprocess, sys, time, psutil\n"
+ "c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'])\n"
+ "ids = {n: {'pid': p.pid, 'created': p.create_time()} for n, p in"
+ " (('kernel', psutil.Process()), ('cell_child', psutil.Process(c.pid)))}\n"
+ "open(%r + '.tmp', 'w').write(json.dumps(ids)); os.replace(%r + '.tmp', %r)\n"
+ "time.sleep(120)\n") % (receipt, receipt, receipt)
+k.proc.stdin.write((json.dumps({"id": "x", "code": code}) + "\n").encode())
+k.proc.stdin.flush()
+while True:
+ time.sleep(1)
+'''
+
+
+def _alive(identity):
+ """Zombie-aware: a killed non-child lingers as a zombie until its reaper collects it."""
+ try:
+ proc = psutil.Process(identity["pid"])
+ return proc.create_time() == identity["created"] and proc.status() != psutil.STATUS_ZOMBIE
+ except psutil.NoSuchProcess:
+ return False
+
+
+def _read(path):
+ wait_until(path.exists, timeout=30, what=f"kernel receipt {path.name}")
+ return json.loads(path.read_text(encoding="utf-8-sig"))
+
+
+# The kernel and its cell child reparent to init once their owner dies, so cleaning them up leaves
+# this test's subtree; real signal delivery to a real orphan is the point of the test.
[email protected]_system_guard_bypass
+def test_owner_sigkill_reaps_the_kernel_group_not_a_live_owners(tmp_path):
+ script = tmp_path / "disposable owner.py"
+ script.write_text(_OWNER, encoding="utf-8")
+ env = dict(os.environ, HOME=str(tmp_path), HERMES_HOME=str(tmp_path / "home"),
+ PYTHONPATH=str(Path(__file__).resolve().parents[2]))
+ ids = {}
+ with (tmp_path / "owners.log").open("w", encoding="utf-8") as log:
+ owners = {role: subprocess.Popen([sys.executable, str(script), str(tmp_path), role], env=env,
+ stdin=subprocess.DEVNULL, stdout=log, stderr=log)
+ for role in ("control", "victim")}
+ try:
+ ids = {role: _read(tmp_path / f"{role}.json") for role in owners}
+ for role in owners:
+ assert os.getpgid(ids[role]["cell_child"]["pid"]) == ids[role]["kernel"]["pid"], (
+ "the cell's subprocess left the kernel's group", role)
+
+ owners["victim"].kill() # SIGKILL: no atexit, no finally, nothing of ours runs
+ owners["victim"].wait(timeout=10)
+
+ wait_until(lambda: not any(_alive(i) for i in ids["victim"].values()), timeout=15,
+ what="the dead owner's kernel and its cell child to exit")
+ assert all(_alive(i) for i in ids["control"].values()), (
+ "a live owner's kernel group was killed", ids["control"])
+ finally:
+ for identity in (i for group in ids.values() for i in group.values()):
+ if _alive(identity):
+ psutil.Process(identity["pid"]).kill()
+ for owner in owners.values():
+ if owner.poll() is None:
+ owner.kill()
+ owner.wait(timeout=10)
+
+
+def _kernel_with_cell_child(monkeypatch, tmp_path, child_argv):
+ """A real kernel (spawn registration captured) running a cell that spawned *child_argv*."""
+ from hermes_cli import process_identity
+ from tools.code_kernel import SessionKernel, _spawn
+
+ calls = []
+ for verb, name in (("register", "own_process_groups"), ("unregister", "release_process_groups")):
+ monkeypatch.setattr(process_identity, name, lambda pgids, verb=verb: calls.append((verb, list(pgids))))
+ monkeypatch.setattr(process_identity, "register_child", lambda pid, purpose: None)
+ kernel = SessionKernel(("graceful",))
+ _spawn(kernel, task_id="graceful", child_python=sys.executable, child_cwd="",
+ sandbox_tools=frozenset(), max_tool_calls=1)
+ receipt = tmp_path / "cell_child.json"
+ code = ("import json, os, subprocess, time, psutil\n"
+ f"c = subprocess.Popen({child_argv!r})\n"
+ "ids = {'pid': c.pid, 'created': psutil.Process(c.pid).create_time()}\n"
+ f"open({str(receipt)!r} + '.tmp', 'w').write(json.dumps(ids))\n"
+ f"os.replace({str(receipt)!r} + '.tmp', {str(receipt)!r})\n"
+ "time.sleep(60)\n")
+ kernel.proc.stdin.write((json.dumps({"id": "x", "code": code}) + "\n").encode())
+ kernel.proc.stdin.flush()
+ return kernel, _read(receipt), calls
+
+
+def _kill(identity):
+ if _alive(identity):
+ psutil.Process(identity["pid"]).kill()
+ wait_until(lambda: not _alive(identity), what="cell child cleanup")
+
+
+def test_teardown_releases_the_kernels_group(monkeypatch, tmp_path):
+ kernel, child, calls = _kernel_with_cell_child(
+ monkeypatch, tmp_path, [sys.executable, "-c", "import time; time.sleep(60)"])
+ try:
+ assert calls == [("register", [kernel.proc.pid])]
+ kernel.teardown()
+ assert not _alive(child), "teardown left the cell's subprocess running"
+ assert calls[-1] == ("unregister", [kernel.proc.pid]), (
+ "a kernel group this process tore down itself stayed registered with the owner-death supervisor")
+ finally:
+ kernel.teardown()
+ _kill(child)
+
+
+# The surviving cell child is orphaned once its kernel exits; killing it leaves the test subtree.
[email protected]_system_guard_bypass
+def test_teardown_keeps_covering_a_cell_child_that_survived_it(monkeypatch, tmp_path):
+ # Ignores the teardown's SIGTERM; the kernel itself exits on it, so no SIGKILL escalation follows.
+ kernel, child, calls = _kernel_with_cell_child(
+ monkeypatch, tmp_path, ["sh", "-c", "trap '' TERM; exec sleep 60"])
+ try:
+ kernel.teardown()
+ assert _alive(child), "precondition: the TERM-ignoring cell child survived teardown"
+ assert calls == [("register", [kernel.proc.pid])], (
+ "the group was released while a cell child was still in it")
+ finally:
+ _kill(child)
diff --git a/tools/code_kernel.py b/tools/code_kernel.py
index 64c32569e62..4cead5af654 100644
--- a/tools/code_kernel.py
+++ b/tools/code_kernel.py
@@ -88,39 +88,6 @@ _CAPTURE_LIMIT = {_RUNNER_CAPTURE_BYTES}
_SPILL_DIR = os.environ.get("HERMES_KERNEL_SPILL_DIR", "")
_SPILL_CAP = {5_000_000}
_PARENT_PROCESS_HANDLE = os.environ.pop("HERMES_KERNEL_PARENT_PROCESS_HANDLE", "")
-_PARENT_DEATH_FD = os.environ.pop("HERMES_KERNEL_PARENT_DEATH_FD", "")
-
-
-def _start_parent_death_pipe_watchdog():
- """POSIX twin of the Windows handle watchdog: exit when the parent dies.
-
- The host holds the only write end of an inherited pipe; a blocking read
- returns EOF the instant the host exits by ANY means (SIGKILL, OOM, crash),
- exactly like the MCP death supervisor. Stdin EOF alone is not enough: the
- main loop only sees it between cells, so a kernel SIGKILLed mid-cell
- outlived its host. Not PR_SET_PDEATHSIG — that is bound to the spawning
- THREAD, and kernels are spawned from per-cell threads that exit.
- """
- global _PARENT_DEATH_FD
- raw_fd = _PARENT_DEATH_FD
- _PARENT_DEATH_FD = ""
- if sys.platform == "win32" or not raw_fd:
- return
- try:
- fd = int(raw_fd)
- os.set_inheritable(fd, False)
- except (OSError, ValueError):
- return
-
- def _wait():
- try:
- while os.read(fd, 1):
- pass
- except OSError:
- pass
- os._exit(0)
-
- threading.Thread(target=_wait, name="hermes-parent-watchdog", daemon=True).start()
def _start_parent_process_watchdog():
@@ -174,7 +141,6 @@ def _start_parent_process_watchdog():
_start_parent_process_watchdog()
-_start_parent_death_pipe_watchdog()
_real_stdout = sys.stdout
@@ -312,7 +278,6 @@ class SessionKernel:
self.sock_path: Optional[str] = None
self.server_sock: Optional[socket.socket] = None
self.stop_event = threading.Event()
- self.death_pipe_w: Optional[int] = None
self.tool_call_log: list = []
self.cell_log_start = 0
self.tool_call_counter: list[int] = [0]
@@ -338,15 +303,11 @@ class SessionKernel:
def teardown(self) -> None:
self.stop_event.set()
- if self.death_pipe_w is not None:
- try:
- os.close(self.death_pipe_w)
- except OSError:
- pass
- self.death_pipe_w = None
if self.alive():
from tools.code_execution_tool import _kill_process_group
_kill_process_group(self.proc, escalate=True)
+ if self.proc is not None and not _IS_WINDOWS:
+ _release_group_once_empty(self.proc)
sock, self.server_sock = self.server_sock, None
try:
if sock is not None:
@@ -360,6 +321,16 @@ class SessionKernel:
shutil.rmtree(self.tmpdir, ignore_errors=True)
+def _release_group_once_empty(proc: subprocess.Popen) -> None:
+ """Hand a torn-down kernel's group back from the owner-death supervisor once nothing in it is
+ left. SIGTERMed cell children exit (and get reaped by init) asynchronously, so wait briefly; a
+ cell child that ignored the SIGTERM stays covered, and the next own/release call prunes it."""
+ from hermes_cli.process_identity import release_process_groups
+ from tools.environments.local import _wait_for_group_exit
+ if _wait_for_group_exit(proc, proc.pid, 1.0):
+ release_process_groups([proc.pid])
+
+
class KernelRegistry:
"""Key -> kernel map plus its lock (shared with the remote registry). Kernels are popped
under the lock and torn down outside it — teardown may block on the child or the transport."""
@@ -626,29 +597,24 @@ def _spawn(kernel: SessionKernel, *, child_python: str, child_cwd: str,
child_env["HERMES_KERNEL_SPILL_DIR"] = kernel.tmpdir
# Generated client reconnects after the RPC server's 300s idle timeout between cells.
child_env["HERMES_RPC_PERSISTENT"] = "1"
- # Parent-death watchdog plumbing: Windows inherits a SYNCHRONIZE handle to this process; POSIX
- # inherits the read end of a pipe whose only write end we hold (EOF == host gone, any cause).
+ # Owner death: Windows inherits a SYNCHRONIZE handle to this process for the runner's watchdog.
parent_handle, close_handle, startupinfo = _parent_process_handle(child_env) if _IS_WINDOWS else (None, None, None)
- death_r: Optional[int] = None
- pass_fds: tuple[int, ...] = ()
- if not _IS_WINDOWS:
- death_r, kernel.death_pipe_w = os.pipe()
- child_env["HERMES_KERNEL_PARENT_DEATH_FD"] = str(death_r)
- pass_fds = (death_r,)
+ # POSIX: spawn_owned makes the kernel lead its own group, which everything a cell spawns joins,
+ # and hands that group to the owner-death supervisor: the whole group dies the moment this
+ # process dies, by any cause. A kernel-only death pipe let a cell's subprocesses outlive the host.
+ from hermes_cli.process_identity import spawn_owned
try:
- kernel.proc = subprocess.Popen(
- [child_python, os.path.join(kernel.tmpdir, "hermes_kernel_runner.py")],
+ kernel.proc = spawn_owned(
+ [child_python, os.path.join(kernel.tmpdir, "hermes_kernel_runner.py")], purpose="code-kernel",
# Strict mode passes an empty cwd: the kernel's staging dir plays the per-call tmpdir's role.
- cwd=child_cwd or kernel.tmpdir, env=child_env, start_new_session=True,
+ cwd=child_cwd or kernel.tmpdir, env=child_env,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE,
creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0,
- close_fds=True, pass_fds=pass_fds, startupinfo=startupinfo,
+ close_fds=True, startupinfo=startupinfo,
)
finally:
if parent_handle and close_handle is not None:
close_handle(parent_handle)
- if death_r is not None:
- os.close(death_r)
# Deliberately NOT propagate_context_to_thread: that would freeze the spawning cell's
# context/callbacks into the server thread for life. Authority is rebound per cell.
for target, args in ((_rpc_forever, (kernel, max_tool_calls, sandbox_tools)),