Skip to content

The agent that grows with you

fix(local-runtime): stop() after a router crash stops its surviving model children and keeps them covered

kshitijk4poor8bc78140fc73
diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.py
index f82923d474f..9a12d8e28b4 100644
--- a/hermes_cli/local_runtime/supervisor.py
+++ b/hermes_cli/local_runtime/supervisor.py
@@ -27,7 +27,7 @@ from pathlib import Path
from hermes_cli.local_runtime.binaries import runtimes_root
from hermes_cli.local_runtime.endpoint import child_query
from hermes_cli.local_runtime.processes import server_child_env, spawn_server
-from hermes_cli.process_identity import release_process_groups
+from hermes_cli.process_identity import stop_owned_group_survivors
logger = logging.getLogger(__name__)
@@ -297,10 +297,11 @@ class LlamaServerSupervisor:
if self._job is not None:
self._job.close()
self._job = None
- if self.proc is not None:
- # Stopped by us: the owner-death supervisor has nothing left to cover. Only
- # after a terminate that returned, so a tree we failed to kill stays covered.
- release_process_groups([self.proc.pid])
+ if self.proc is not None and sys.platform != "win32":
+ # A router that crashed (the watcher's restart backoff) can leave its model children
+ # running in its group: stop them, then release the group only once it is empty, so
+ # a tree we failed to kill stays covered by the owner-death supervisor.
+ stop_owned_group_survivors(self.proc.pid, proc=self.proc)
# Retain state: deleting it could race a replacement publication.
if self._log_handle:
self._log_handle.close()
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index a6473434cb3..0739923e0ff 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -18,6 +18,7 @@ import json
import logging
import os
import platform
+import signal
import subprocess
import sys
import threading
@@ -885,6 +886,31 @@ def release_process_group_once_empty(pgid: int, *, proc=None, timeout: float = 1
time.sleep(0.05)
+def stop_owned_group_survivors(pgid: int, *, proc=None, grace: float = 3.0) -> bool:
+ """Stop what is left of owned group ``pgid`` after its leader exited, then release it; True once
+ released. Members that outlived a crashed leader (a router's model children) keep the group, and
+ so its number, alive: TERM, wait ``grace``, KILL. Only groups this process registered are
+ signalled, so a number never registered here (or already released) is never touched."""
+ if os.name != "posix":
+ return False
+ with _supervisor_lock:
+ owned = pgid in _supervised_pgids
+ if owned:
+ for sig, wait in ((signal.SIGTERM, grace), (signal.SIGKILL, 1.0)):
+ if release_process_group_once_empty(pgid, proc=proc, timeout=0):
+ return True
+ try:
+ os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only, returned above
+ except ProcessLookupError:
+ pass
+ except OSError:
+ return False # not ours to signal: keep covering it
+ if release_process_group_once_empty(pgid, proc=proc, timeout=wait):
+ return True
+ return False
+ return release_process_group_once_empty(pgid, proc=proc, timeout=0)
+
+
def spawn_owned(argv, *, purpose: str, **popen_kwargs) -> subprocess.Popen:
"""Start a long-lived child that dies with this process: the one way to own a child.
diff --git a/tests/hermes_cli/test_local_runtime_owned_router.py b/tests/hermes_cli/test_local_runtime_owned_router.py
index 1ba46a072ee..58d0a77514c 100644
--- a/tests/hermes_cli/test_local_runtime_owned_router.py
+++ b/tests/hermes_cli/test_local_runtime_owned_router.py
@@ -4,16 +4,18 @@ All processes and homes are disposable; the owner is a real Hermes process impor
``spawn_server(owned=True)`` path, and it is SIGKILLed so no Python cleanup of ours can run.
"""
+import contextlib
import json
import os
from pathlib import Path
import subprocess
import sys
-import time
import psutil
import pytest
+from tests.waiting import wait_until
+
pytestmark = pytest.mark.platforms("posix")
_SCRIPT = r'''
@@ -40,17 +42,8 @@ else:
'''
-def _wait(predicate, timeout=15.0):
- deadline = time.monotonic() + timeout
- while time.monotonic() < deadline:
- if predicate():
- return True
- time.sleep(0.05)
- return bool(predicate())
-
-
def _read(path):
- assert _wait(path.exists), f"missing child receipt: {path}"
+ wait_until(path.exists, timeout=15, what=f"child receipt {path}")
return json.loads(path.read_text(encoding="utf-8-sig"))
@@ -88,8 +81,8 @@ def test_owner_sigkill_reaps_router_tree_not_an_external_control(tmp_path):
launchers[1].kill() # SIGKILL the owner: no atexit, no finally, nothing of ours runs
launchers[1].wait(timeout=10)
- assert _wait(lambda: not _alive(ids["router"]) and not _alive(ids["grandchild"])), (
- "router tree survived kill -9 of its owner", ids)
+ wait_until(lambda: not _alive(ids["router"]) and not _alive(ids["grandchild"]), timeout=15,
+ what=f"router tree to die after kill -9 of its owner: {ids}")
assert _alive(ids["control"]), "an unrelated process was killed"
finally:
for identity in ids.values():
@@ -101,21 +94,51 @@ def test_owner_sigkill_reaps_router_tree_not_an_external_control(tmp_path):
def test_a_graceful_stop_releases_the_routers_group(monkeypatch, tmp_path):
+ from hermes_cli import process_identity
from hermes_cli.local_runtime import supervisor
- released = []
monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)
- monkeypatch.setattr(supervisor, "release_process_groups",
- lambda pgids: released.append(("unregister", list(pgids))))
sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59998)
sup.proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"],
stdin=subprocess.DEVNULL, start_new_session=True)
+ process_identity.own_process_groups([sup.proc.pid])
try:
sup.stop()
assert sup.proc.poll() is not None, "stop() left the router running"
- assert released == [("unregister", [sup.proc.pid])], (
+ assert sup.proc.pid not in process_identity._supervised_pgids, (
"a router this process stopped itself stayed registered with the owner-death supervisor")
finally:
if sup.proc.poll() is None:
sup.proc.kill()
sup.proc.wait(timeout=10)
+ process_identity.release_process_groups([sup.proc.pid])
+
+
+def test_stop_after_a_router_crash_still_stops_its_surviving_model_child(monkeypatch, tmp_path):
+ """The watcher's restart backoff: the router died, its model child (same group) still runs."""
+ from hermes_cli import process_identity
+ from hermes_cli.local_runtime import supervisor
+
+ monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)
+ sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59997)
+ receipt = tmp_path / "model.pid"
+ sup.proc = subprocess.Popen( # detached-spawn: ok — test router stand-in; its group is killed below
+ [sys.executable, "-c",
+ "import subprocess, sys; c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)']);"
+ f" open({str(receipt)!r}, 'w').write(str(c.pid))"],
+ stdin=subprocess.DEVNULL, start_new_session=True)
+ process_identity.own_process_groups([sup.proc.pid])
+ pgid = sup.proc.pid
+ try:
+ sup.proc.wait(timeout=30) # the router "crashed"; the model child is still in its group
+ model = psutil.Process(int(wait_until(lambda: receipt.exists() and receipt.read_text(encoding="utf-8"),
+ what="model child receipt")))
+ assert os.getpgid(model.pid) == pgid
+ sup.stop()
+ wait_until(lambda: not model.is_running() or model.status() == psutil.STATUS_ZOMBIE, timeout=10,
+ what="stop() to stop the crashed router's surviving model child")
+ assert pgid not in process_identity._supervised_pgids
+ finally:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(pgid, 9) # windows-footgun: ok — posix-marked test
+ process_identity.release_process_groups([pgid])