The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(local-runtime): the llama-server tree dies with its owner on POSIX too (orphans_after_30s 2 -> 0)
kshitijk4poor97bef249accc
diff --git a/hermes_cli/local_runtime/processes.py b/hermes_cli/local_runtime/processes.pyindex 4e3196e22c7..b62fc3801a6 100644--- a/hermes_cli/local_runtime/processes.py+++ b/hermes_cli/local_runtime/processes.py@@ -1,4 +1,4 @@-"""Contain each managed Windows router tree without adopting the owner process."""+"""Contain each managed router tree without adopting the owner process."""from __future__ import annotations@@ -107,15 +107,27 @@ def server_child_env(base_env: Mapping[str, str]) -> dict[str, str]:}-def spawn_server(cmd, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:+def spawn_server(cmd, *, owned: bool = False, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:"""Start a router, returning its process and an owner-held containment handle.Keep the job until shutdown and call close() to terminate the entire tree.- Windows closes it automatically if the owner dies. Other hosts retain Popen's- ordinary behavior. Assignment happens before the child's first instruction.+ Windows closes it automatically if the owner dies. On POSIX ``owned=True`` gives+ the same guarantee: the router leads its own process group (its model children+ stay in it) and that group goes to this process's owner-death supervisor, which+ kills it the moment the owner dies by any cause (#126631). Without it Popen's+ ordinary behavior applies (this is also the generic bounded-probe spawner, whose+ short-lived children need no supervision). Assignment happens before the+ child's first instruction."""if sys.platform != "win32":- return subprocess.Popen(cmd, **kwargs), None+ if not owned:+ return subprocess.Popen(cmd, **kwargs), None+ from hermes_cli.process_identity import register_child, update_supervised_pgids++ proc = subprocess.Popen(cmd, start_new_session=True, **kwargs)+ update_supervised_pgids("register", [proc.pid])+ register_child(proc.pid, "llama-server")+ return proc, Nonejob = _WindowsJob()proc = Nonetry:diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.pyindex d2ef1b50d78..51344cc711f 100644--- a/hermes_cli/local_runtime/supervisor.py+++ b/hermes_cli/local_runtime/supervisor.py@@ -27,6 +27,7 @@ from pathlib import Pathfrom hermes_cli.local_runtime.binaries import runtimes_rootfrom hermes_cli.local_runtime.endpoint import child_queryfrom hermes_cli.local_runtime.processes import server_child_env, spawn_server+from hermes_cli.process_identity import update_supervised_pgidslogger = logging.getLogger(__name__)@@ -203,8 +204,9 @@ class LlamaServerSupervisor:self._log_handle = open(self.log_path, "a", encoding="utf-8", errors="replace")self._log_handle.write(f"\n# spawn: {cmd}\n")self._log_handle.flush()- # list-args, never a shell: spaced paths (user homes) must survive.- self.proc, self._job = spawn_server(cmd, stdout=self._log_handle, stderr=subprocess.STDOUT,+ # list-args, never a shell: spaced paths (user homes) must survive. owned: the router+ # tree dies with this process on every OS, however this process dies (#126631).+ self.proc, self._job = spawn_server(cmd, owned=True, stdout=self._log_handle, stderr=subprocess.STDOUT,cwd=str(exe.parent), env=server_child_env(os.environ))logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)# State goes down at SPAWN, not after health: endpoint resolution treats a@@ -295,6 +297,10 @@ class LlamaServerSupervisor:if self._job is not None:self._job.close()self._job = None+ if self.proc is not None:+ # Stopped by us: the owner-death supervisor has nothing left to cover. Only+ # after a terminate that returned, so a tree we failed to kill stays covered.+ update_supervised_pgids("unregister", [self.proc.pid])# Retain state: deleting it could race a replacement publication.if self._log_handle:self._log_handle.close()diff --git a/tests/hermes_cli/test_local_runtime_owned_router.py b/tests/hermes_cli/test_local_runtime_owned_router.pynew file mode 100644index 00000000000..ddaf2baa510--- /dev/null+++ b/tests/hermes_cli/test_local_runtime_owned_router.py@@ -0,0 +1,121 @@+"""POSIX owned-router containment: the llama-server tree dies with its owner, however the owner dies.++All processes and homes are disposable; the owner is a real Hermes process importing the real+``spawn_server(owned=True)`` path, and it is SIGKILLed so no Python cleanup of ours can run.+"""++import json+import os+from pathlib import Path+import subprocess+import sys+import time++import psutil+import pytest++pytestmark = pytest.mark.platforms("posix")++_SCRIPT = r'''+import json, subprocess, sys, time+from pathlib import Path+import psutil+root = Path(sys.argv[1])+role = sys.argv[2]+p = psutil.Process()+(root / (role + '.tmp')).write_text(json.dumps({'pid': p.pid, 'created': p.create_time()}))+(root / (role + '.tmp')).replace(root / (role + '.json'))+if role == 'owner':+ from hermes_cli.local_runtime.processes import spawn_server+ proc, _ = spawn_server([sys.executable, __file__, str(root), 'router'], owned=True)+ while True:+ time.sleep(1)+elif role == 'router':+ # llama.cpp's router spawns its model children with posix_spawn and no setpgid, so they+ # stay in the router's group; the stand-in does the same.+ subprocess.Popen([sys.executable, __file__, str(root), 'grandchild'])+ time.sleep(90)+else:+ time.sleep(90)+'''+++def _wait(predicate, timeout=15.0):+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ if predicate():+ return True+ time.sleep(0.05)+ return bool(predicate())+++def _read(path):+ assert _wait(path.exists), f"missing child receipt: {path}"+ return json.loads(path.read_text(encoding="utf-8-sig"))+++def _alive(identity):+ """Zombie-aware: a killed non-child lingers as a zombie until its reaper collects it."""+ try:+ proc = psutil.Process(identity["pid"])+ return proc.create_time() == identity["created"] and proc.status() != psutil.STATUS_ZOMBIE+ except psutil.NoSuchProcess:+ return False+++def _kill(identity):+ if _alive(identity):+ psutil.Process(identity["pid"]).kill()+++# The router and its model child reparent to init once the owner dies, so cleaning them up+# leaves this test's subtree; real signal delivery to a real orphan is the point of the test.[email protected]_system_guard_bypass+def test_owner_sigkill_reaps_router_tree_not_an_external_control(tmp_path):+ script = tmp_path / "disposable server.py"+ script.write_text(_SCRIPT, encoding="utf-8")+ env = dict(os.environ, HERMES_HOME=str(tmp_path / "home"),+ PYTHONPATH=str(Path(__file__).resolve().parents[2]))+ with (tmp_path / "children.log").open("w", encoding="utf-8") as log:+ launchers = [subprocess.Popen([sys.executable, str(script), str(tmp_path), role],+ env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=log)+ for role in ("control", "owner")]+ ids = {}+ try:+ ids = {role: _read(tmp_path / f"{role}.json") for role in ("owner", "router", "grandchild", "control")}+ assert os.getpgid(ids["grandchild"]["pid"]) == ids["router"]["pid"], "model child left the router's group"++ launchers[1].kill() # SIGKILL the owner: no atexit, no finally, nothing of ours runs+ launchers[1].wait(timeout=10)++ assert _wait(lambda: not _alive(ids["router"]) and not _alive(ids["grandchild"])), (+ "router tree survived kill -9 of its owner", ids)+ assert _alive(ids["control"]), "an unrelated process was killed"+ finally:+ for identity in ids.values():+ _kill(identity)+ for launcher in launchers:+ if launcher.poll() is None:+ launcher.kill()+ launcher.wait(timeout=10)+++def test_a_graceful_stop_releases_the_routers_group(monkeypatch, tmp_path):+ from hermes_cli.local_runtime import supervisor++ released = []+ monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)+ monkeypatch.setattr(supervisor, "update_supervised_pgids",+ lambda verb, pgids: released.append((verb, list(pgids))))+ sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59998)+ sup.proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"],+ stdin=subprocess.DEVNULL, start_new_session=True)+ try:+ sup.stop()+ assert sup.proc.poll() is not None, "stop() left the router running"+ assert released == [("unregister", [sup.proc.pid])], (+ "a router this process stopped itself stayed registered with the owner-death supervisor")+ finally:+ if sup.proc.poll() is None:+ sup.proc.kill()+ sup.proc.wait(timeout=10)