Skip to content

The agent that grows with you

fix(process): a family's stop path releases its group only once the group is empty

kshitijk4poor57644ec0004b
diff --git a/agent/verify/runner.py b/agent/verify/runner.py
index be12e5ac3f5..314b74742c8 100644
--- a/agent/verify/runner.py
+++ b/agent/verify/runner.py
@@ -19,7 +19,7 @@ from pathlib import Path
from typing import Any, Callable
from agent.verify.recipes import Recipe
-from hermes_cli.process_identity import release_process_groups, spawn_owned
+from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
DEFAULT_PHASE_TIMEOUT = 600.0
DEFAULT_READY_TIMEOUT = 60.0
@@ -182,7 +182,7 @@ def _run_start_phase(
finally:
_terminate_process_group(proc)
if proc.poll() is not None:
- release_process_groups([proc.pid])
+ release_process_group_once_empty(proc.pid, proc=proc)
try:
output = proc.stdout.read() or "" if proc.stdout is not None else ""
except (OSError, ValueError):
diff --git a/hermes_cli/web_routers/messaging.py b/hermes_cli/web_routers/messaging.py
index 8c061ef6065..e9041ccbd3e 100644
--- a/hermes_cli/web_routers/messaging.py
+++ b/hermes_cli/web_routers/messaging.py
@@ -25,7 +25,7 @@ from fastapi import APIRouter, HTTPException
from gateway.status import resolve_gateway_liveness, retained_gateway_state
from hermes_cli._subprocess_compat import windows_hide_flags
from hermes_cli.config import OPTIONAL_ENV_VARS, get_env_path
-from hermes_cli.process_identity import release_process_groups, spawn_owned
+from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
from hermes_constants import get_process_hermes_home
from hermes_cli.web_deps import LateState, late
from hermes_cli.web_server_gateway import _restart_gateway_after
@@ -456,7 +456,7 @@ def _terminate_whatsapp_pairing(proc: subprocess.Popen | None) -> None:
proc.wait(timeout=3)
if proc.poll() is not None:
# Stopped; one we failed to stop stays covered by the owner-death supervisor.
- release_process_groups([proc.pid])
+ release_process_group_once_empty(proc.pid, proc=proc)
def _fail_whatsapp_pairing(pairing_id: str, error: str, *, proc=None, unless=_WHATSAPP_ONBOARDING_TERMINAL_STATUSES) -> None:
@@ -510,7 +510,7 @@ def _watch_whatsapp_pairing(pairing_id: str, proc: subprocess.Popen) -> None:
if record.status not in {"cancelled", "expired"}:
_apply_pairing_event(record, payload)
returncode = proc.wait()
- release_process_groups([proc.pid])
+ release_process_group_once_empty(proc.pid, proc=proc)
except Exception as exc:
_fail_whatsapp_pairing(pairing_id, str(exc), proc=proc)
return
diff --git a/plugins/platforms/raft/adapter.py b/plugins/platforms/raft/adapter.py
index 4765f0fc8b8..d9381cc1b96 100644
--- a/plugins/platforms/raft/adapter.py
+++ b/plugins/platforms/raft/adapter.py
@@ -40,7 +40,7 @@ from gateway.config import Platform, PlatformConfig
from gateway.platforms.base import BasePlatformAdapter, SendResult, merge_pending_message_event
from gateway.platforms.event import MessageEvent, MessageType
from gateway.platforms._shared import coerce_port, profile_scoped as _profile_scoped
-from hermes_cli.process_identity import release_process_groups, spawn_owned
+from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
logger = logging.getLogger(__name__)
@@ -404,7 +404,7 @@ class RaftAdapter(BasePlatformAdapter):
logger.exception("[raft] Error stopping bridge")
if proc.poll() is not None:
# Stopped by us; a bridge we failed to stop stays covered by the owner-death supervisor.
- release_process_groups([proc.pid])
+ release_process_group_once_empty(proc.pid, proc=proc)
async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[dict[str, Any]] = None) -> SendResult:
diff --git a/plugins/platforms/whatsapp/adapter.py b/plugins/platforms/whatsapp/adapter.py
index 1756fccb031..fd47ce9569b 100644
--- a/plugins/platforms/whatsapp/adapter.py
+++ b/plugins/platforms/whatsapp/adapter.py
@@ -19,7 +19,7 @@ from gateway.platforms._shared import (
)
from gateway.platforms.send_targets import StandaloneSendPolicy, regex_stripped_parser
from hermes_cli._subprocess_compat import windows_detach_flags
-from hermes_cli.process_identity import release_process_groups, spawn_owned
+from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
from hermes_constants import (find_node_executable, get_hermes_dir, with_hermes_node_path)
_IS_WINDOWS = platform.system() == "Windows"
@@ -865,7 +865,7 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter):
if not _IS_WINDOWS and self._bridge_process.poll() is not None:
# Stopped by us; a bridge we failed to stop stays covered by the owner-death supervisor.
# (POSIX only: Windows has no supervised group, and its stop path stays as on main.)
- release_process_groups([self._bridge_process.pid])
+ release_process_group_once_empty(self._bridge_process.pid, timeout=0)
if self._bridge_process or not getattr(self, "_runtime_status_platform_key", ""):
_unlink_quietly(self._session_path / "bridge.pid")
await cancel_task(self._poll_task)
diff --git a/tests/hermes_cli/test_owned_agent_subprocs.py b/tests/hermes_cli/test_owned_agent_subprocs.py
index abd37accbaf..80f8831fd45 100644
--- a/tests/hermes_cli/test_owned_agent_subprocs.py
+++ b/tests/hermes_cli/test_owned_agent_subprocs.py
@@ -6,8 +6,10 @@ while the live owner's tree is untouched. On origin/main every one of these tree
forever: plain Popen children reparent to init, and detached ones lead their own session.
"""
+import contextlib
import json
import os
+import signal
import subprocess
import sys
from pathlib import Path
@@ -15,6 +17,8 @@ from pathlib import Path
import psutil
import pytest
+from hermes_cli import process_identity
+
from tests.waiting import wait_until
pytestmark = pytest.mark.platforms("posix")
@@ -182,7 +186,7 @@ async def test_whatsapp_disconnect_releases_the_bridge_it_stopped(monkeypatch):
from plugins.platforms.whatsapp import adapter as wa
released = []
- monkeypatch.setattr(wa, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ monkeypatch.setattr(process_identity, "release_process_groups", lambda pgids: released.append(list(pgids)))
bridge = _sleeper()
from tests.gateway.test_whatsapp_connect import _make_adapter
@@ -201,7 +205,7 @@ def test_raft_stop_releases_the_bridge_it_stopped(monkeypatch):
from plugins.platforms.raft import adapter as raft
released = []
- monkeypatch.setattr(raft, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ monkeypatch.setattr(process_identity, "release_process_groups", lambda pgids: released.append(list(pgids)))
bridge = _sleeper()
a = raft.RaftAdapter.__new__(raft.RaftAdapter)
a._bridge_process = bridge
@@ -218,7 +222,7 @@ def test_verify_start_phase_owns_then_releases_the_app_group(monkeypatch, tmp_pa
from agent.verify.recipes import Recipe
released = []
- monkeypatch.setattr(runner, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ monkeypatch.setattr(process_identity, "release_process_groups", lambda pgids: released.append(list(pgids)))
result = runner._run_start_phase(Recipe(name="x", start="sleep 30", port=1), tmp_path, 0.3)
assert not result.ready
assert len(owned) == 1 and released == [owned]
@@ -228,7 +232,7 @@ def test_tts_command_owns_then_releases_its_group_after_success_and_idle_kill(mo
from tools import tts_command_provider as tts
released = []
- monkeypatch.setattr(tts, "release_process_groups", lambda pgids: released.extend(pgids))
+ monkeypatch.setattr(process_identity, "release_process_groups", lambda pgids: released.extend(pgids))
assert tts.run_command_provider("echo hi", 10.0).stdout.strip() == "hi"
with pytest.raises(subprocess.TimeoutExpired):
tts.run_command_provider("sleep 30", 0.3)
@@ -239,7 +243,7 @@ def test_whatsapp_pairing_terminate_releases_the_process_it_stopped(monkeypatch)
from hermes_cli.web_routers import messaging
released = []
- monkeypatch.setattr(messaging, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ monkeypatch.setattr(process_identity, "release_process_groups", lambda pgids: released.append(list(pgids)))
pairing = _sleeper()
try:
messaging._terminate_whatsapp_pairing(pairing)
@@ -247,3 +251,27 @@ def test_whatsapp_pairing_terminate_releases_the_process_it_stopped(monkeypatch)
assert released == [[pairing.pid]]
finally:
_reap(pairing)
+
+
+def test_a_stopped_bridge_whose_child_survived_stays_covered(monkeypatch, tmp_path):
+ """Leader exit is not proof the group is empty: a helper that ignores TERM keeps it alive, and
+ releasing that group would let the helper outlive a later kill -9 of this process."""
+ from plugins.platforms.raft import adapter as raft
+
+ released = []
+ monkeypatch.setattr(process_identity, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ receipt = tmp_path / "helper.pid"
+ bridge = subprocess.Popen( # detached-spawn: ok — test bridge stand-in; its group is killed in finally
+ ["sh", "-c", f"(trap '' TERM; echo $(exec sh -c 'echo $PPID') > {receipt}; exec sleep 60) & exec sleep 60"],
+ stdin=subprocess.DEVNULL, start_new_session=True)
+ a = raft.RaftAdapter.__new__(raft.RaftAdapter)
+ a._bridge_process = bridge
+ try:
+ wait_until(receipt.exists, what="helper receipt")
+ a._stop_bridge()
+ assert bridge.poll() is not None, "_stop_bridge() left the bridge running"
+ assert released == [], "a group with a live helper was released from the owner-death supervisor"
+ finally:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(bridge.pid, signal.SIGKILL) # windows-footgun: ok — posix-marked file
+ _reap(bridge)
diff --git a/tests/tools/test_browser_use_cli_owned_daemon.py b/tests/tools/test_browser_use_cli_owned_daemon.py
index f726f343478..96874c2c852 100644
--- a/tests/tools/test_browser_use_cli_owned_daemon.py
+++ b/tests/tools/test_browser_use_cli_owned_daemon.py
@@ -52,7 +52,9 @@ def fake_daemon(tmp_path):
@pytest.fixture
def registered(monkeypatch):
calls = []
- for verb, name in (("register", "own_process_groups"), ("unregister", "release_process_groups")):
+ monkeypatch.setattr(browser_use_cli, "release_process_group_once_empty",
+ lambda pgid, **kw: calls.append(("unregister", [pgid])))
+ for verb, name in (("register", "own_process_groups"),):
monkeypatch.setattr(browser_use_cli, name, lambda pgids, verb=verb: calls.append((verb, list(pgids))))
monkeypatch.setattr(browser_use_cli, "register_child", lambda pid, purpose: None)
monkeypatch.setattr(browser_use_cli, "_owned_daemon_groups", {})
diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py
index d8fae81c179..5a0c348cf23 100644
--- a/tools/browser_use_cli.py
+++ b/tools/browser_use_cli.py
@@ -23,7 +23,7 @@ import time
from pathlib import Path
from typing import Any, Callable, Dict, List, NamedTuple, Optional
-from hermes_cli.process_identity import own_process_groups, register_child, release_process_groups
+from hermes_cli.process_identity import own_process_groups, register_child, release_process_group_once_empty
from hermes_constants import get_hermes_home, hermes_home_key
from tools.browser_harness_dirs import (daemon_env, daemon_identity, fit_default_socket_dir,
namespace_harness_dirs)
@@ -643,7 +643,7 @@ def _reload_daemons(daemons) -> None: # each ``daemon_identity`` daemon, with i
with _owned_daemon_groups_lock:
pgid = _owned_daemon_groups.pop(daemon, None)
if pgid is not None:
- release_process_groups([pgid])
+ release_process_group_once_empty(pgid) # a daemon --reload failed to stop stays covered
# Daemon identity (``daemon_identity``: its dirs + BU_NAME) -> process group of the harness daemon this
diff --git a/tools/computer_use/cua_backend_daemon.py b/tools/computer_use/cua_backend_daemon.py
index c0157513259..60e87861f40 100644
--- a/tools/computer_use/cua_backend_daemon.py
+++ b/tools/computer_use/cua_backend_daemon.py
@@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple
from tools.computer_use import cua_backend_driver as _driver
from tools.computer_use.permissions import CUA_DRIVER_BUNDLE_ID
-from hermes_cli.process_identity import release_process_groups, spawn_owned
+from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
logger = logging.getLogger("tools.computer_use.cua_backend")
@@ -215,7 +215,7 @@ class _EmbeddedCuaDaemon:
stderr=subprocess.DEVNULL, env=self._sanitized_env(), swallow=_QUIET_ERRORS)
if process is not None:
_wait_or_kill(process)
- release_process_groups([process.pid]) # stopped it ourselves (a no-op for the macOS `open` launcher)
+ release_process_group_once_empty(process.pid, proc=process) # stopped it ourselves (a no-op for the macOS `open` launcher)
if sys.platform != "win32" and os.path.exists(self.socket_path):
with contextlib.suppress(OSError):
os.remove(self.socket_path)
diff --git a/tools/tts_command_provider.py b/tools/tts_command_provider.py
index 41953b6d337..f0a357e68d0 100644
--- a/tools/tts_command_provider.py
+++ b/tools/tts_command_provider.py
@@ -23,7 +23,7 @@ from functools import partial
from pathlib import Path
from typing import Any, Dict, FrozenSet, Optional
-from hermes_cli.process_identity import release_process_groups, spawn_owned
+from hermes_cli.process_identity import release_process_group_once_empty, spawn_owned
from utils import is_truthy_value
@@ -233,7 +233,7 @@ def run_command_provider(
if chunk:
chunks[name].append(chunk)
if proc.poll() is not None:
- release_process_groups([proc.pid]) # a tree we failed to stop stays covered
+ release_process_group_once_empty(proc.pid, proc=proc) # a tree we failed to stop stays covered
stdout, stderr = "".join(chunks["stdout"]), "".join(chunks["stderr"])
if timed_out:
raise subprocess.TimeoutExpired(command, timeout, output=stdout, stderr=stderr) from (