The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(execute_code): a kernel's cell subprocesses die with the Hermes process that ran them (orphans_after_10s 3 -> 0)
kshitijk4pooreb834d16f2bf
diff --git a/tests/tools/test_code_kernel.py b/tests/tools/test_code_kernel.pyindex 9846656f0ed..f05e4e861ec 100644--- a/tests/tools/test_code_kernel.py+++ b/tests/tools/test_code_kernel.py@@ -107,8 +107,9 @@ class TestKernelLifecycle(unittest.TestCase):def test_kernel_exits_when_its_backend_parent_dies(self):"""A kernel must not outlive the host that spawned it, even when thehost dies without cleanup (SIGKILL/OOM/crash). Windows: inherited- SYNCHRONIZE handle; POSIX: inherited death pipe. Both are proven the- same way — kill the host mid-cell, the kernel is gone within seconds."""+ SYNCHRONIZE handle; POSIX: the kernel's group is registered with the+ owner-death supervisor. Both are proven the same way — kill the host+ mid-cell, the kernel is gone within seconds."""import psutilrepo_root = str(Path(__file__).resolve().parents[2])diff --git a/tests/tools/test_code_kernel_owned_group.py b/tests/tools/test_code_kernel_owned_group.pynew file mode 100644index 00000000000..d77b59f10c2--- /dev/null+++ b/tests/tools/test_code_kernel_owned_group.py@@ -0,0 +1,150 @@+"""POSIX session kernels: the kernel's whole process group dies with its owner, however the owner dies.++The owners are real host processes importing the real ``tools.code_kernel._spawn``; each starts a kernel+whose cell spawns a subprocess of its own (it stays in the kernel's group, as every cell child does).+One owner is SIGKILLed so no Python cleanup of ours can run; the other is the live-owner control.+"""++import json+import os+from pathlib import Path+import subprocess+import sys++import psutil+import pytest++from tests.waiting import wait_until++pytestmark = pytest.mark.platforms("posix")++_OWNER = r'''+import json, sys, time+from pathlib import Path+root, role = Path(sys.argv[1]), sys.argv[2]+from tools.code_kernel import SessionKernel, _spawn+k = SessionKernel((role,))+_spawn(k, task_id=role, child_python=sys.executable, child_cwd="", sandbox_tools=frozenset(),+ max_tool_calls=1)+receipt = str(root / (role + ".json"))+code = (+ "import json, os, subprocess, sys, time, psutil\n"+ "c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'])\n"+ "ids = {n: {'pid': p.pid, 'created': p.create_time()} for n, p in"+ " (('kernel', psutil.Process()), ('cell_child', psutil.Process(c.pid)))}\n"+ "open(%r + '.tmp', 'w').write(json.dumps(ids)); os.replace(%r + '.tmp', %r)\n"+ "time.sleep(120)\n") % (receipt, receipt, receipt)+k.proc.stdin.write((json.dumps({"id": "x", "code": code}) + "\n").encode())+k.proc.stdin.flush()+while True:+ time.sleep(1)+'''+++def _alive(identity):+ """Zombie-aware: a killed non-child lingers as a zombie until its reaper collects it."""+ try:+ proc = psutil.Process(identity["pid"])+ return proc.create_time() == identity["created"] and proc.status() != psutil.STATUS_ZOMBIE+ except psutil.NoSuchProcess:+ return False+++def _read(path):+ wait_until(path.exists, timeout=30, what=f"kernel receipt {path.name}")+ return json.loads(path.read_text(encoding="utf-8-sig"))+++# The kernel and its cell child reparent to init once their owner dies, so cleaning them up leaves+# this test's subtree; real signal delivery to a real orphan is the point of the test.[email protected]_system_guard_bypass+def test_owner_sigkill_reaps_the_kernel_group_not_a_live_owners(tmp_path):+ script = tmp_path / "disposable owner.py"+ script.write_text(_OWNER, encoding="utf-8")+ env = dict(os.environ, HOME=str(tmp_path), HERMES_HOME=str(tmp_path / "home"),+ PYTHONPATH=str(Path(__file__).resolve().parents[2]))+ ids = {}+ with (tmp_path / "owners.log").open("w", encoding="utf-8") as log:+ owners = {role: subprocess.Popen([sys.executable, str(script), str(tmp_path), role], env=env,+ stdin=subprocess.DEVNULL, stdout=log, stderr=log)+ for role in ("control", "victim")}+ try:+ ids = {role: _read(tmp_path / f"{role}.json") for role in owners}+ for role in owners:+ assert os.getpgid(ids[role]["cell_child"]["pid"]) == ids[role]["kernel"]["pid"], (+ "the cell's subprocess left the kernel's group", role)++ owners["victim"].kill() # SIGKILL: no atexit, no finally, nothing of ours runs+ owners["victim"].wait(timeout=10)++ wait_until(lambda: not any(_alive(i) for i in ids["victim"].values()), timeout=15,+ what="the dead owner's kernel and its cell child to exit")+ assert all(_alive(i) for i in ids["control"].values()), (+ "a live owner's kernel group was killed", ids["control"])+ finally:+ for identity in (i for group in ids.values() for i in group.values()):+ if _alive(identity):+ psutil.Process(identity["pid"]).kill()+ for owner in owners.values():+ if owner.poll() is None:+ owner.kill()+ owner.wait(timeout=10)+++def _kernel_with_cell_child(monkeypatch, tmp_path, child_argv):+ """A real kernel (spawn registration captured) running a cell that spawned *child_argv*."""+ from hermes_cli import process_identity+ from tools.code_kernel import SessionKernel, _spawn++ calls = []+ for verb, name in (("register", "own_process_groups"), ("unregister", "release_process_groups")):+ monkeypatch.setattr(process_identity, name, lambda pgids, verb=verb: calls.append((verb, list(pgids))))+ monkeypatch.setattr(process_identity, "register_child", lambda pid, purpose: None)+ kernel = SessionKernel(("graceful",))+ _spawn(kernel, task_id="graceful", child_python=sys.executable, child_cwd="",+ sandbox_tools=frozenset(), max_tool_calls=1)+ receipt = tmp_path / "cell_child.json"+ code = ("import json, os, subprocess, time, psutil\n"+ f"c = subprocess.Popen({child_argv!r})\n"+ "ids = {'pid': c.pid, 'created': psutil.Process(c.pid).create_time()}\n"+ f"open({str(receipt)!r} + '.tmp', 'w').write(json.dumps(ids))\n"+ f"os.replace({str(receipt)!r} + '.tmp', {str(receipt)!r})\n"+ "time.sleep(60)\n")+ kernel.proc.stdin.write((json.dumps({"id": "x", "code": code}) + "\n").encode())+ kernel.proc.stdin.flush()+ return kernel, _read(receipt), calls+++def _kill(identity):+ if _alive(identity):+ psutil.Process(identity["pid"]).kill()+ wait_until(lambda: not _alive(identity), what="cell child cleanup")+++def test_teardown_releases_the_kernels_group(monkeypatch, tmp_path):+ kernel, child, calls = _kernel_with_cell_child(+ monkeypatch, tmp_path, [sys.executable, "-c", "import time; time.sleep(60)"])+ try:+ assert calls == [("register", [kernel.proc.pid])]+ kernel.teardown()+ assert not _alive(child), "teardown left the cell's subprocess running"+ assert calls[-1] == ("unregister", [kernel.proc.pid]), (+ "a kernel group this process tore down itself stayed registered with the owner-death supervisor")+ finally:+ kernel.teardown()+ _kill(child)+++# The surviving cell child is orphaned once its kernel exits; killing it leaves the test subtree.[email protected]_system_guard_bypass+def test_teardown_keeps_covering_a_cell_child_that_survived_it(monkeypatch, tmp_path):+ # Ignores the teardown's SIGTERM; the kernel itself exits on it, so no SIGKILL escalation follows.+ kernel, child, calls = _kernel_with_cell_child(+ monkeypatch, tmp_path, ["sh", "-c", "trap '' TERM; exec sleep 60"])+ try:+ kernel.teardown()+ assert _alive(child), "precondition: the TERM-ignoring cell child survived teardown"+ assert calls == [("register", [kernel.proc.pid])], (+ "the group was released while a cell child was still in it")+ finally:+ _kill(child)diff --git a/tools/code_kernel.py b/tools/code_kernel.pyindex 64c32569e62..4cead5af654 100644--- a/tools/code_kernel.py+++ b/tools/code_kernel.py@@ -88,39 +88,6 @@ _CAPTURE_LIMIT = {_RUNNER_CAPTURE_BYTES}_SPILL_DIR = os.environ.get("HERMES_KERNEL_SPILL_DIR", "")_SPILL_CAP = {5_000_000}_PARENT_PROCESS_HANDLE = os.environ.pop("HERMES_KERNEL_PARENT_PROCESS_HANDLE", "")-_PARENT_DEATH_FD = os.environ.pop("HERMES_KERNEL_PARENT_DEATH_FD", "")---def _start_parent_death_pipe_watchdog():- """POSIX twin of the Windows handle watchdog: exit when the parent dies.-- The host holds the only write end of an inherited pipe; a blocking read- returns EOF the instant the host exits by ANY means (SIGKILL, OOM, crash),- exactly like the MCP death supervisor. Stdin EOF alone is not enough: the- main loop only sees it between cells, so a kernel SIGKILLed mid-cell- outlived its host. Not PR_SET_PDEATHSIG — that is bound to the spawning- THREAD, and kernels are spawned from per-cell threads that exit.- """- global _PARENT_DEATH_FD- raw_fd = _PARENT_DEATH_FD- _PARENT_DEATH_FD = ""- if sys.platform == "win32" or not raw_fd:- return- try:- fd = int(raw_fd)- os.set_inheritable(fd, False)- except (OSError, ValueError):- return-- def _wait():- try:- while os.read(fd, 1):- pass- except OSError:- pass- os._exit(0)-- threading.Thread(target=_wait, name="hermes-parent-watchdog", daemon=True).start()def _start_parent_process_watchdog():@@ -174,7 +141,6 @@ def _start_parent_process_watchdog():_start_parent_process_watchdog()-_start_parent_death_pipe_watchdog()_real_stdout = sys.stdout@@ -312,7 +278,6 @@ class SessionKernel:self.sock_path: Optional[str] = Noneself.server_sock: Optional[socket.socket] = Noneself.stop_event = threading.Event()- self.death_pipe_w: Optional[int] = Noneself.tool_call_log: list = []self.cell_log_start = 0self.tool_call_counter: list[int] = [0]@@ -338,15 +303,11 @@ class SessionKernel:def teardown(self) -> None:self.stop_event.set()- if self.death_pipe_w is not None:- try:- os.close(self.death_pipe_w)- except OSError:- pass- self.death_pipe_w = Noneif self.alive():from tools.code_execution_tool import _kill_process_group_kill_process_group(self.proc, escalate=True)+ if self.proc is not None and not _IS_WINDOWS:+ _release_group_once_empty(self.proc)sock, self.server_sock = self.server_sock, Nonetry:if sock is not None:@@ -360,6 +321,16 @@ class SessionKernel:shutil.rmtree(self.tmpdir, ignore_errors=True)+def _release_group_once_empty(proc: subprocess.Popen) -> None:+ """Hand a torn-down kernel's group back from the owner-death supervisor once nothing in it is+ left. SIGTERMed cell children exit (and get reaped by init) asynchronously, so wait briefly; a+ cell child that ignored the SIGTERM stays covered, and the next own/release call prunes it."""+ from hermes_cli.process_identity import release_process_groups+ from tools.environments.local import _wait_for_group_exit+ if _wait_for_group_exit(proc, proc.pid, 1.0):+ release_process_groups([proc.pid])++class KernelRegistry:"""Key -> kernel map plus its lock (shared with the remote registry). Kernels are poppedunder the lock and torn down outside it — teardown may block on the child or the transport."""@@ -626,29 +597,24 @@ def _spawn(kernel: SessionKernel, *, child_python: str, child_cwd: str,child_env["HERMES_KERNEL_SPILL_DIR"] = kernel.tmpdir# Generated client reconnects after the RPC server's 300s idle timeout between cells.child_env["HERMES_RPC_PERSISTENT"] = "1"- # Parent-death watchdog plumbing: Windows inherits a SYNCHRONIZE handle to this process; POSIX- # inherits the read end of a pipe whose only write end we hold (EOF == host gone, any cause).+ # Owner death: Windows inherits a SYNCHRONIZE handle to this process for the runner's watchdog.parent_handle, close_handle, startupinfo = _parent_process_handle(child_env) if _IS_WINDOWS else (None, None, None)- death_r: Optional[int] = None- pass_fds: tuple[int, ...] = ()- if not _IS_WINDOWS:- death_r, kernel.death_pipe_w = os.pipe()- child_env["HERMES_KERNEL_PARENT_DEATH_FD"] = str(death_r)- pass_fds = (death_r,)+ # POSIX: spawn_owned makes the kernel lead its own group, which everything a cell spawns joins,+ # and hands that group to the owner-death supervisor: the whole group dies the moment this+ # process dies, by any cause. A kernel-only death pipe let a cell's subprocesses outlive the host.+ from hermes_cli.process_identity import spawn_ownedtry:- kernel.proc = subprocess.Popen(- [child_python, os.path.join(kernel.tmpdir, "hermes_kernel_runner.py")],+ kernel.proc = spawn_owned(+ [child_python, os.path.join(kernel.tmpdir, "hermes_kernel_runner.py")], purpose="code-kernel",# Strict mode passes an empty cwd: the kernel's staging dir plays the per-call tmpdir's role.- cwd=child_cwd or kernel.tmpdir, env=child_env, start_new_session=True,+ cwd=child_cwd or kernel.tmpdir, env=child_env,stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE,creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0,- close_fds=True, pass_fds=pass_fds, startupinfo=startupinfo,+ close_fds=True, startupinfo=startupinfo,)finally:if parent_handle and close_handle is not None:close_handle(parent_handle)- if death_r is not None:- os.close(death_r)# Deliberately NOT propagate_context_to_thread: that would freeze the spawning cell's# context/callbacks into the server thread for life. Authority is rebound per cell.for target, args in ((_rpc_forever, (kernel, max_tool_calls, sandbox_tools)),