The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(local-runtime): a crashed router's empty group is released before the restart backoff
kshitijk4poorf84eb589ea76
diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.pyindex 53c158cec7e..6fd9e6868df 100644--- a/hermes_cli/local_runtime/supervisor.py+++ b/hermes_cli/local_runtime/supervisor.py@@ -27,7 +27,7 @@ from pathlib import Pathfrom hermes_cli.local_runtime.binaries import runtimes_rootfrom hermes_cli.local_runtime.endpoint import child_queryfrom hermes_cli.local_runtime.processes import server_child_env, spawn_server-from hermes_cli.process_identity import stop_owned_group_survivors+from hermes_cli.process_identity import release_process_group_once_empty, stop_owned_group_survivorslogger = logging.getLogger(__name__)@@ -269,6 +269,9 @@ class LlamaServerSupervisor:continueif self._stopping:return+ # An already-empty group is released now, not after the backoff: its number is free for+ # reuse meanwhile, and the survivor stop below would then signal a stranger's group.+ release_process_group_once_empty(proc.pid, proc=proc, timeout=0)backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)if self._stop_event.wait(backoff):diff --git a/tests/hermes_cli/test_local_runtime_owned_router.py b/tests/hermes_cli/test_local_runtime_owned_router.pyindex bafcbae6b3c..f0f80ce15aa 100644--- a/tests/hermes_cli/test_local_runtime_owned_router.py+++ b/tests/hermes_cli/test_local_runtime_owned_router.py@@ -175,3 +175,30 @@ def test_a_crash_restart_stops_the_old_routers_survivors_and_releases_its_group(with contextlib.suppress(ProcessLookupError):os.killpg(old.pid, 9) # windows-footgun: ok — posix-marked testprocess_identity.release_process_groups([old.pid])+++def test_a_crashed_routers_empty_group_is_released_before_the_restart_backoff(monkeypatch, tmp_path):+ """Kept registered through the backoff, the freed number could be reused by a stranger's group,+ which the restart's survivor stop would then TERM and KILL."""+ from hermes_cli import process_identity+ from hermes_cli.local_runtime import supervisor++ monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)+ sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59995)+ old = subprocess.Popen([sys.executable, "-c", "pass"], # detached-spawn: ok — test router stand-in; exits at once+ stdin=subprocess.DEVNULL, start_new_session=True)+ process_identity.own_process_groups([old.pid])+ registered_during_backoff = []++ def backoff_wait(_seconds):+ registered_during_backoff.append(old.pid in process_identity._supervised_pgids)+ return True # stop() arrived during the backoff: ends the watch++ try:+ old.wait(timeout=30) # the router crashed and left nothing in its group+ sup.proc = old+ monkeypatch.setattr(sup._stop_event, "wait", backoff_wait)+ sup._watch()+ assert registered_during_backoff == [False], "the empty group stayed registered through the backoff"+ finally:+ process_identity.release_process_groups([old.pid])