Skip to content

The agent that grows with you

fix(local-runtime): a crashed router's empty group is released before the restart backoff

kshitijk4poorf84eb589ea76
diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.py
index 53c158cec7e..6fd9e6868df 100644
--- a/hermes_cli/local_runtime/supervisor.py
+++ b/hermes_cli/local_runtime/supervisor.py
@@ -27,7 +27,7 @@ from pathlib import Path
from hermes_cli.local_runtime.binaries import runtimes_root
from hermes_cli.local_runtime.endpoint import child_query
from hermes_cli.local_runtime.processes import server_child_env, spawn_server
-from hermes_cli.process_identity import stop_owned_group_survivors
+from hermes_cli.process_identity import release_process_group_once_empty, stop_owned_group_survivors
logger = logging.getLogger(__name__)
@@ -269,6 +269,9 @@ class LlamaServerSupervisor:
continue
if self._stopping:
return
+ # An already-empty group is released now, not after the backoff: its number is free for
+ # reuse meanwhile, and the survivor stop below would then signal a stranger's group.
+ release_process_group_once_empty(proc.pid, proc=proc, timeout=0)
backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)
if self._stop_event.wait(backoff):
diff --git a/tests/hermes_cli/test_local_runtime_owned_router.py b/tests/hermes_cli/test_local_runtime_owned_router.py
index bafcbae6b3c..f0f80ce15aa 100644
--- a/tests/hermes_cli/test_local_runtime_owned_router.py
+++ b/tests/hermes_cli/test_local_runtime_owned_router.py
@@ -175,3 +175,30 @@ def test_a_crash_restart_stops_the_old_routers_survivors_and_releases_its_group(
with contextlib.suppress(ProcessLookupError):
os.killpg(old.pid, 9) # windows-footgun: ok — posix-marked test
process_identity.release_process_groups([old.pid])
+
+
+def test_a_crashed_routers_empty_group_is_released_before_the_restart_backoff(monkeypatch, tmp_path):
+ """Kept registered through the backoff, the freed number could be reused by a stranger's group,
+ which the restart's survivor stop would then TERM and KILL."""
+ from hermes_cli import process_identity
+ from hermes_cli.local_runtime import supervisor
+
+ monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)
+ sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59995)
+ old = subprocess.Popen([sys.executable, "-c", "pass"], # detached-spawn: ok — test router stand-in; exits at once
+ stdin=subprocess.DEVNULL, start_new_session=True)
+ process_identity.own_process_groups([old.pid])
+ registered_during_backoff = []
+
+ def backoff_wait(_seconds):
+ registered_during_backoff.append(old.pid in process_identity._supervised_pgids)
+ return True # stop() arrived during the backoff: ends the watch
+
+ try:
+ old.wait(timeout=30) # the router crashed and left nothing in its group
+ sup.proc = old
+ monkeypatch.setattr(sup._stop_event, "wait", backoff_wait)
+ sup._watch()
+ assert registered_during_backoff == [False], "the empty group stayed registered through the backoff"
+ finally:
+ process_identity.release_process_groups([old.pid])