The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(local-runtime): stop() after a router crash stops its surviving model children and keeps them covered
kshitijk4poor8bc78140fc73
diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.pyindex f82923d474f..9a12d8e28b4 100644--- a/hermes_cli/local_runtime/supervisor.py+++ b/hermes_cli/local_runtime/supervisor.py@@ -27,7 +27,7 @@ from pathlib import Pathfrom hermes_cli.local_runtime.binaries import runtimes_rootfrom hermes_cli.local_runtime.endpoint import child_queryfrom hermes_cli.local_runtime.processes import server_child_env, spawn_server-from hermes_cli.process_identity import release_process_groups+from hermes_cli.process_identity import stop_owned_group_survivorslogger = logging.getLogger(__name__)@@ -297,10 +297,11 @@ class LlamaServerSupervisor:if self._job is not None:self._job.close()self._job = None- if self.proc is not None:- # Stopped by us: the owner-death supervisor has nothing left to cover. Only- # after a terminate that returned, so a tree we failed to kill stays covered.- release_process_groups([self.proc.pid])+ if self.proc is not None and sys.platform != "win32":+ # A router that crashed (the watcher's restart backoff) can leave its model children+ # running in its group: stop them, then release the group only once it is empty, so+ # a tree we failed to kill stays covered by the owner-death supervisor.+ stop_owned_group_survivors(self.proc.pid, proc=self.proc)# Retain state: deleting it could race a replacement publication.if self._log_handle:self._log_handle.close()diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.pyindex a6473434cb3..0739923e0ff 100644--- a/hermes_cli/process_identity.py+++ b/hermes_cli/process_identity.py@@ -18,6 +18,7 @@ import jsonimport loggingimport osimport platform+import signalimport subprocessimport sysimport threading@@ -885,6 +886,31 @@ def release_process_group_once_empty(pgid: int, *, proc=None, timeout: float = 1time.sleep(0.05)+def stop_owned_group_survivors(pgid: int, *, proc=None, grace: float = 3.0) -> bool:+ """Stop what is left of owned group ``pgid`` after its leader exited, then release it; True once+ released. Members that outlived a crashed leader (a router's model children) keep the group, and+ so its number, alive: TERM, wait ``grace``, KILL. Only groups this process registered are+ signalled, so a number never registered here (or already released) is never touched."""+ if os.name != "posix":+ return False+ with _supervisor_lock:+ owned = pgid in _supervised_pgids+ if owned:+ for sig, wait in ((signal.SIGTERM, grace), (signal.SIGKILL, 1.0)):+ if release_process_group_once_empty(pgid, proc=proc, timeout=0):+ return True+ try:+ os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only, returned above+ except ProcessLookupError:+ pass+ except OSError:+ return False # not ours to signal: keep covering it+ if release_process_group_once_empty(pgid, proc=proc, timeout=wait):+ return True+ return False+ return release_process_group_once_empty(pgid, proc=proc, timeout=0)++def spawn_owned(argv, *, purpose: str, **popen_kwargs) -> subprocess.Popen:"""Start a long-lived child that dies with this process: the one way to own a child.diff --git a/tests/hermes_cli/test_local_runtime_owned_router.py b/tests/hermes_cli/test_local_runtime_owned_router.pyindex 1ba46a072ee..58d0a77514c 100644--- a/tests/hermes_cli/test_local_runtime_owned_router.py+++ b/tests/hermes_cli/test_local_runtime_owned_router.py@@ -4,16 +4,18 @@ All processes and homes are disposable; the owner is a real Hermes process impor``spawn_server(owned=True)`` path, and it is SIGKILLed so no Python cleanup of ours can run."""+import contextlibimport jsonimport osfrom pathlib import Pathimport subprocessimport sys-import timeimport psutilimport pytest+from tests.waiting import wait_until+pytestmark = pytest.mark.platforms("posix")_SCRIPT = r'''@@ -40,17 +42,8 @@ else:'''-def _wait(predicate, timeout=15.0):- deadline = time.monotonic() + timeout- while time.monotonic() < deadline:- if predicate():- return True- time.sleep(0.05)- return bool(predicate())--def _read(path):- assert _wait(path.exists), f"missing child receipt: {path}"+ wait_until(path.exists, timeout=15, what=f"child receipt {path}")return json.loads(path.read_text(encoding="utf-8-sig"))@@ -88,8 +81,8 @@ def test_owner_sigkill_reaps_router_tree_not_an_external_control(tmp_path):launchers[1].kill() # SIGKILL the owner: no atexit, no finally, nothing of ours runslaunchers[1].wait(timeout=10)- assert _wait(lambda: not _alive(ids["router"]) and not _alive(ids["grandchild"])), (- "router tree survived kill -9 of its owner", ids)+ wait_until(lambda: not _alive(ids["router"]) and not _alive(ids["grandchild"]), timeout=15,+ what=f"router tree to die after kill -9 of its owner: {ids}")assert _alive(ids["control"]), "an unrelated process was killed"finally:for identity in ids.values():@@ -101,21 +94,51 @@ def test_owner_sigkill_reaps_router_tree_not_an_external_control(tmp_path):def test_a_graceful_stop_releases_the_routers_group(monkeypatch, tmp_path):+ from hermes_cli import process_identityfrom hermes_cli.local_runtime import supervisor- released = []monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)- monkeypatch.setattr(supervisor, "release_process_groups",- lambda pgids: released.append(("unregister", list(pgids))))sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59998)sup.proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"],stdin=subprocess.DEVNULL, start_new_session=True)+ process_identity.own_process_groups([sup.proc.pid])try:sup.stop()assert sup.proc.poll() is not None, "stop() left the router running"- assert released == [("unregister", [sup.proc.pid])], (+ assert sup.proc.pid not in process_identity._supervised_pgids, ("a router this process stopped itself stayed registered with the owner-death supervisor")finally:if sup.proc.poll() is None:sup.proc.kill()sup.proc.wait(timeout=10)+ process_identity.release_process_groups([sup.proc.pid])+++def test_stop_after_a_router_crash_still_stops_its_surviving_model_child(monkeypatch, tmp_path):+ """The watcher's restart backoff: the router died, its model child (same group) still runs."""+ from hermes_cli import process_identity+ from hermes_cli.local_runtime import supervisor++ monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)+ sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59997)+ receipt = tmp_path / "model.pid"+ sup.proc = subprocess.Popen( # detached-spawn: ok — test router stand-in; its group is killed below+ [sys.executable, "-c",+ "import subprocess, sys; c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)']);"+ f" open({str(receipt)!r}, 'w').write(str(c.pid))"],+ stdin=subprocess.DEVNULL, start_new_session=True)+ process_identity.own_process_groups([sup.proc.pid])+ pgid = sup.proc.pid+ try:+ sup.proc.wait(timeout=30) # the router "crashed"; the model child is still in its group+ model = psutil.Process(int(wait_until(lambda: receipt.exists() and receipt.read_text(encoding="utf-8"),+ what="model child receipt")))+ assert os.getpgid(model.pid) == pgid+ sup.stop()+ wait_until(lambda: not model.is_running() or model.status() == psutil.STATUS_ZOMBIE, timeout=10,+ what="stop() to stop the crashed router's surviving model child")+ assert pgid not in process_identity._supervised_pgids+ finally:+ with contextlib.suppress(ProcessLookupError):+ os.killpg(pgid, 9) # windows-footgun: ok — posix-marked test+ process_identity.release_process_groups([pgid])