Skip to content

The agent that grows with you

fix(local-runtime): the llama-server tree dies with its owner on POSIX too (orphans_after_30s 2 -> 0)

kshitijk4poor97bef249accc
diff --git a/hermes_cli/local_runtime/processes.py b/hermes_cli/local_runtime/processes.py
index 4e3196e22c7..b62fc3801a6 100644
--- a/hermes_cli/local_runtime/processes.py
+++ b/hermes_cli/local_runtime/processes.py
@@ -1,4 +1,4 @@
-"""Contain each managed Windows router tree without adopting the owner process."""
+"""Contain each managed router tree without adopting the owner process."""
from __future__ import annotations
@@ -107,15 +107,27 @@ def server_child_env(base_env: Mapping[str, str]) -> dict[str, str]:
}
-def spawn_server(cmd, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:
+def spawn_server(cmd, *, owned: bool = False, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:
"""Start a router, returning its process and an owner-held containment handle.
Keep the job until shutdown and call close() to terminate the entire tree.
- Windows closes it automatically if the owner dies. Other hosts retain Popen's
- ordinary behavior. Assignment happens before the child's first instruction.
+ Windows closes it automatically if the owner dies. On POSIX ``owned=True`` gives
+ the same guarantee: the router leads its own process group (its model children
+ stay in it) and that group goes to this process's owner-death supervisor, which
+ kills it the moment the owner dies by any cause (#126631). Without it Popen's
+ ordinary behavior applies (this is also the generic bounded-probe spawner, whose
+ short-lived children need no supervision). Assignment happens before the
+ child's first instruction.
"""
if sys.platform != "win32":
- return subprocess.Popen(cmd, **kwargs), None
+ if not owned:
+ return subprocess.Popen(cmd, **kwargs), None
+ from hermes_cli.process_identity import register_child, update_supervised_pgids
+
+ proc = subprocess.Popen(cmd, start_new_session=True, **kwargs)
+ update_supervised_pgids("register", [proc.pid])
+ register_child(proc.pid, "llama-server")
+ return proc, None
job = _WindowsJob()
proc = None
try:
diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.py
index d2ef1b50d78..51344cc711f 100644
--- a/hermes_cli/local_runtime/supervisor.py
+++ b/hermes_cli/local_runtime/supervisor.py
@@ -27,6 +27,7 @@ from pathlib import Path
from hermes_cli.local_runtime.binaries import runtimes_root
from hermes_cli.local_runtime.endpoint import child_query
from hermes_cli.local_runtime.processes import server_child_env, spawn_server
+from hermes_cli.process_identity import update_supervised_pgids
logger = logging.getLogger(__name__)
@@ -203,8 +204,9 @@ class LlamaServerSupervisor:
self._log_handle = open(self.log_path, "a", encoding="utf-8", errors="replace")
self._log_handle.write(f"\n# spawn: {cmd}\n")
self._log_handle.flush()
- # list-args, never a shell: spaced paths (user homes) must survive.
- self.proc, self._job = spawn_server(cmd, stdout=self._log_handle, stderr=subprocess.STDOUT,
+ # list-args, never a shell: spaced paths (user homes) must survive. owned: the router
+ # tree dies with this process on every OS, however this process dies (#126631).
+ self.proc, self._job = spawn_server(cmd, owned=True, stdout=self._log_handle, stderr=subprocess.STDOUT,
cwd=str(exe.parent), env=server_child_env(os.environ))
logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)
# State goes down at SPAWN, not after health: endpoint resolution treats a
@@ -295,6 +297,10 @@ class LlamaServerSupervisor:
if self._job is not None:
self._job.close()
self._job = None
+ if self.proc is not None:
+ # Stopped by us: the owner-death supervisor has nothing left to cover. Only
+ # after a terminate that returned, so a tree we failed to kill stays covered.
+ update_supervised_pgids("unregister", [self.proc.pid])
# Retain state: deleting it could race a replacement publication.
if self._log_handle:
self._log_handle.close()
diff --git a/tests/hermes_cli/test_local_runtime_owned_router.py b/tests/hermes_cli/test_local_runtime_owned_router.py
new file mode 100644
index 00000000000..ddaf2baa510
--- /dev/null
+++ b/tests/hermes_cli/test_local_runtime_owned_router.py
@@ -0,0 +1,121 @@
+"""POSIX owned-router containment: the llama-server tree dies with its owner, however the owner dies.
+
+All processes and homes are disposable; the owner is a real Hermes process importing the real
+``spawn_server(owned=True)`` path, and it is SIGKILLed so no Python cleanup of ours can run.
+"""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+
+import psutil
+import pytest
+
+pytestmark = pytest.mark.platforms("posix")
+
+_SCRIPT = r'''
+import json, subprocess, sys, time
+from pathlib import Path
+import psutil
+root = Path(sys.argv[1])
+role = sys.argv[2]
+p = psutil.Process()
+(root / (role + '.tmp')).write_text(json.dumps({'pid': p.pid, 'created': p.create_time()}))
+(root / (role + '.tmp')).replace(root / (role + '.json'))
+if role == 'owner':
+ from hermes_cli.local_runtime.processes import spawn_server
+ proc, _ = spawn_server([sys.executable, __file__, str(root), 'router'], owned=True)
+ while True:
+ time.sleep(1)
+elif role == 'router':
+ # llama.cpp's router spawns its model children with posix_spawn and no setpgid, so they
+ # stay in the router's group; the stand-in does the same.
+ subprocess.Popen([sys.executable, __file__, str(root), 'grandchild'])
+ time.sleep(90)
+else:
+ time.sleep(90)
+'''
+
+
+def _wait(predicate, timeout=15.0):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if predicate():
+ return True
+ time.sleep(0.05)
+ return bool(predicate())
+
+
+def _read(path):
+ assert _wait(path.exists), f"missing child receipt: {path}"
+ return json.loads(path.read_text(encoding="utf-8-sig"))
+
+
+def _alive(identity):
+ """Zombie-aware: a killed non-child lingers as a zombie until its reaper collects it."""
+ try:
+ proc = psutil.Process(identity["pid"])
+ return proc.create_time() == identity["created"] and proc.status() != psutil.STATUS_ZOMBIE
+ except psutil.NoSuchProcess:
+ return False
+
+
+def _kill(identity):
+ if _alive(identity):
+ psutil.Process(identity["pid"]).kill()
+
+
+# The router and its model child reparent to init once the owner dies, so cleaning them up
+# leaves this test's subtree; real signal delivery to a real orphan is the point of the test.
[email protected]_system_guard_bypass
+def test_owner_sigkill_reaps_router_tree_not_an_external_control(tmp_path):
+ script = tmp_path / "disposable server.py"
+ script.write_text(_SCRIPT, encoding="utf-8")
+ env = dict(os.environ, HERMES_HOME=str(tmp_path / "home"),
+ PYTHONPATH=str(Path(__file__).resolve().parents[2]))
+ with (tmp_path / "children.log").open("w", encoding="utf-8") as log:
+ launchers = [subprocess.Popen([sys.executable, str(script), str(tmp_path), role],
+ env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=log)
+ for role in ("control", "owner")]
+ ids = {}
+ try:
+ ids = {role: _read(tmp_path / f"{role}.json") for role in ("owner", "router", "grandchild", "control")}
+ assert os.getpgid(ids["grandchild"]["pid"]) == ids["router"]["pid"], "model child left the router's group"
+
+ launchers[1].kill() # SIGKILL the owner: no atexit, no finally, nothing of ours runs
+ launchers[1].wait(timeout=10)
+
+ assert _wait(lambda: not _alive(ids["router"]) and not _alive(ids["grandchild"])), (
+ "router tree survived kill -9 of its owner", ids)
+ assert _alive(ids["control"]), "an unrelated process was killed"
+ finally:
+ for identity in ids.values():
+ _kill(identity)
+ for launcher in launchers:
+ if launcher.poll() is None:
+ launcher.kill()
+ launcher.wait(timeout=10)
+
+
+def test_a_graceful_stop_releases_the_routers_group(monkeypatch, tmp_path):
+ from hermes_cli.local_runtime import supervisor
+
+ released = []
+ monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)
+ monkeypatch.setattr(supervisor, "update_supervised_pgids",
+ lambda verb, pgids: released.append((verb, list(pgids))))
+ sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59998)
+ sup.proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"],
+ stdin=subprocess.DEVNULL, start_new_session=True)
+ try:
+ sup.stop()
+ assert sup.proc.poll() is not None, "stop() left the router running"
+ assert released == [("unregister", [sup.proc.pid])], (
+ "a router this process stopped itself stayed registered with the owner-death supervisor")
+ finally:
+ if sup.proc.poll() is None:
+ sup.proc.kill()
+ sup.proc.wait(timeout=10)