Skip to content

The agent that grows with you

fix(tui): the slash worker's process group dies with the gateway that owns it (orphans_after_30s 1 -> 0)

kshitijk4poorf3ebdfadbdc9
diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py
index 27d4b8cd3f7..0507324f821 100644
--- a/hermes_cli/dashboard_procs.py
+++ b/hermes_cli/dashboard_procs.py
@@ -496,8 +496,8 @@ def _is_detached_session_leader(pid: int, tty: str) -> bool:
Known gap: the turn-isolation ``tui_gateway.compute_host`` and ``slash_worker`` children are
launched the same way (``start_new_session=True``, no tty; ``compute_host`` also holds
``state.db``), so a wedged WAL holder of that class is spared here too. Their exit relies on
- their own ppid watchdogs (``compute_host._parent_guard_loop``,
- ``slash_worker._start_parent_death_watchdog``), not on this sweep. Discriminating by WAL-holder
+ ``compute_host._parent_guard_loop`` and on the gateway's owner-death supervisor
+ (``process_identity.spawn_owned``) respectively, not on this sweep. Discriminating by WAL-holder
identity (``iter_deleted_sqlite_sidecar_holders``) was deliberately not done in this change.
"""
if tty not in _NO_TTY:
diff --git a/tests/tui_gateway/test_served_profile_child_env.py b/tests/tui_gateway/test_served_profile_child_env.py
index 4ef77367618..21768ce5c21 100644
--- a/tests/tui_gateway/test_served_profile_child_env.py
+++ b/tests/tui_gateway/test_served_profile_child_env.py
@@ -72,7 +72,7 @@ def test_slash_worker_child_runs_in_the_served_profiles_env(mux_homes, monkeypat
return 0
with monkeypatch.context() as m: # restored before the probe child spawns through the real Popen
- m.setattr(server.subprocess, "Popen", _Popen)
+ m.setattr(server, "spawn_owned", lambda argv, purpose, **kw: _Popen(argv, **kw))
token = set_hermes_home_override(str(b))
try:
server._SlashWorker("sess", "", profile_home=str(b))
@@ -107,7 +107,7 @@ def test_launch_profile_worker_spawns_with_own_home_after_multiplex_flip(mux_hom
return 0
with monkeypatch.context() as m:
- m.setattr(server.subprocess, "Popen", _Popen)
+ m.setattr(server, "spawn_owned", lambda argv, purpose, **kw: _Popen(argv, **kw))
# Multiplex is already active (mux_homes fixture). profile_home=None = launch-profile session.
server._SlashWorker("sess", "", profile_home=None)
# No UnscopedSecretError: the worker is launched under the launch profile's own home, not routed.
diff --git a/tests/tui_gateway/test_slash_worker_owned.py b/tests/tui_gateway/test_slash_worker_owned.py
new file mode 100644
index 00000000000..b1e14d143f3
--- /dev/null
+++ b/tests/tui_gateway/test_slash_worker_owned.py
@@ -0,0 +1,245 @@
+"""The slash worker and everything it starts in its group die with the gateway that owns it.
+
+``_SlashWorker`` is started through ``process_identity.spawn_owned``: the worker leads its own process
+group and that group goes to the gateway's owner-death supervisor. Before, the worker only had a ppid
+watchdog of its own, so a gateway SIGKILL / OOM left whatever the worker had started (a running quick
+command's shell, its children) alive forever, and a normal close signalled the worker alone.
+
+The live tests drive the real ``_SlashWorker`` and the real worker process under disposable homes; the
+owner is SIGKILLed, so no Python cleanup of ours can run.
+"""
+
+import contextlib
+import importlib.abc
+import importlib.util
+import io
+import json
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+
+import psutil
+import pytest
+
+from tests.waiting import wait_until
+from tui_gateway import server, slash_worker
+
+_REPO = Path(__file__).resolve().parents[2]
+
+# Owner stand-in: the real gateway-side class, optionally running a quick command whose shell stays in
+# the worker's group. Blocks on stdin so a killed pytest leaves no owner behind.
+_OWNER = r'''
+import json, os, sys, threading
+from pathlib import Path
+from tui_gateway import server
+out = Path(sys.argv[1])
+worker = server._SlashWorker("owned-worker-test", "")
+if sys.argv[2] == "quick":
+ threading.Thread(target=lambda: worker.run("/linger"), daemon=True).start()
+(out.with_suffix(".tmp")).write_text(json.dumps({"owner": os.getpid(), "worker": worker.proc.pid}))
+(out.with_suffix(".tmp")).replace(out)
+sys.stdin.read()
+'''
+
+
+def _group(pgid: int) -> list[psutil.Process]:
+ """Live (non-zombie) members of ``pgid``."""
+ members = []
+ for proc in psutil.process_iter():
+ try:
+ if os.getpgid(proc.pid) == pgid and proc.status() != psutil.STATUS_ZOMBIE: # windows-footgun: ok — posix-marked tests
+ members.append(proc)
+ except (OSError, psutil.Error):
+ continue
+ return members
+
+
+def _cmdline(proc: psutil.Process) -> list[str]:
+ try:
+ return proc.cmdline()
+ except psutil.Error:
+ return []
+
+
+def _home(tmp_path: Path, name: str) -> dict:
+ home = tmp_path / name
+ (home / ".hermes").mkdir(parents=True)
+ (home / ".hermes" / "config.yaml").write_text(
+ "quick_commands:\n linger:\n type: exec\n command: sleep 120\n", encoding="utf-8")
+ env = {k: v for k, v in os.environ.items() if not k.endswith("_API_KEY")}
+ env.update(HOME=str(home), HERMES_HOME=str(home / ".hermes"), PYTHONPATH=str(_REPO))
+ return env
+
+
[email protected]_system_guard_bypass # the worker group reparents to init once its owner dies
+def test_owner_sigkill_takes_the_worker_group_and_spares_a_live_owners_worker(tmp_path):
+ script = tmp_path / "owner.py"
+ script.write_text(_OWNER, encoding="utf-8")
+ owners, ids = {}, {}
+ try:
+ for role, mode in (("doomed", "quick"), ("control", "idle")):
+ owners[role] = subprocess.Popen(
+ [sys.executable, str(script), str(tmp_path / f"{role}.json"), mode],
+ env=_home(tmp_path, role), cwd=str(tmp_path), stdin=subprocess.PIPE,
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ for role in owners:
+ receipt = wait_until(lambda role=role: (tmp_path / f"{role}.json").exists()
+ and json.loads((tmp_path / f"{role}.json").read_text(encoding="utf-8")),
+ timeout=60, what=f"{role} owner receipt")
+ ids[role] = receipt["worker"]
+ doomed, control = ids["doomed"], ids["control"]
+ # The quick command's shell runs inside the worker's group; that is what used to survive.
+ def _quick_running():
+ members = _group(doomed)
+ return any(_cmdline(p)[-2:] == ["sleep", "120"] for p in members) and members
+ members = wait_until(_quick_running, timeout=90,
+ what="the quick command running in the doomed worker's group")
+ assert wait_until(lambda: _group(control), timeout=30, what="the control worker running")
+
+ owners["doomed"].kill() # SIGKILL the owner: no atexit, no finally, nothing of ours runs
+ owners["doomed"].wait(timeout=10)
+
+ wait_until(lambda: not _group(doomed), timeout=15,
+ what=f"worker group {doomed} gone after its owner was SIGKILLed: {members}")
+ assert _group(control), "a live owner's slash worker was killed"
+ finally:
+ for pgid in ids.values():
+ for proc in _group(pgid):
+ try:
+ proc.kill()
+ except psutil.Error:
+ pass
+ for owner in owners.values():
+ if owner.poll() is None:
+ owner.kill()
+ owner.communicate(timeout=10)
+
+
+def _group_leader(*shell: str) -> subprocess.Popen:
+ """A stand-in worker leading its own group, with a TERM-ignoring child that stays in it."""
+ return subprocess.Popen(["sh", "-c", *shell], stdin=subprocess.DEVNULL,
+ start_new_session=True) # detached-spawn: ok — test stand-in, killed in finally
+
+
+def test_close_stops_the_whole_group_then_releases_it(monkeypatch):
+ released = []
+ monkeypatch.setattr(server, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ leader = _group_leader("trap '' TERM; sleep 120 & wait")
+ try:
+ wait_until(lambda: len(_group(leader.pid)) >= 2, what="the stand-in's child in its group")
+
+ server._stop_worker_group(leader)
+
+ assert not _group(leader.pid), "close() left a member of the worker's group running"
+ assert released == [[leader.pid]]
+ finally:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(leader.pid, signal.SIGKILL) # windows-footgun: ok — posix-marked test
+ leader.wait(timeout=10)
+
+
+def test_a_worker_that_already_died_is_never_group_signalled(monkeypatch):
+ """A dead leader no longer proves the pgid is ours: leave the group to the supervisor."""
+ released, signalled = [], []
+ monkeypatch.setattr(server, "release_process_groups", lambda pgids: released.append(list(pgids)))
+ leader = _group_leader("sleep 120 & exit 0")
+ try:
+ leader.wait(timeout=10)
+ survivors = wait_until(lambda: _group(leader.pid), what="the leaderless member")
+ real_killpg = os.killpg
+ monkeypatch.setattr(server.os, "killpg",
+ lambda pgid, sig: (signalled.append(sig), real_killpg(pgid, sig))[1])
+
+ server._stop_worker_group(leader)
+
+ assert signalled == [0], "a dead worker's group number was signalled"
+ assert released == [], "a group with a live member was released from the supervisor"
+ assert all(p.is_running() for p in survivors)
+ finally:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(leader.pid, signal.SIGKILL) # windows-footgun: ok — posix-marked test
+
+
+def test_main_imports_cli_before_runtime_prep_and_accepts_a_legacy_parent_pid(monkeypatch):
+ """``cli`` loads ~/.hermes/.env, which MCP discovery in the runtime prep needs. A gateway started
+ before an update still passes ``--parent-pid``; the updated worker must still start."""
+ events = []
+
+ class FakeHermesCLI:
+ def __init__(self, **kwargs):
+ events.append(("cli", kwargs["resume"]))
+
+ class FakeCliFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader):
+ def find_spec(self, name, path=None, target=None):
+ return importlib.util.spec_from_loader(name, self) if name == "cli" else None
+
+ def create_module(self, spec):
+ return None
+
+ def exec_module(self, module):
+ events.append(("cli-import",))
+ module.HermesCLI = FakeHermesCLI
+
+ # setitem first so teardown also removes the stub module the import below inserts.
+ monkeypatch.setitem(sys.modules, "cli", None)
+ monkeypatch.delitem(sys.modules, "cli")
+ monkeypatch.setattr(sys, "meta_path", [FakeCliFinder(), *sys.meta_path])
+ monkeypatch.setattr(slash_worker, "_prepare_slash_worker_runtime", lambda: events.append(("runtime",)))
+ monkeypatch.setattr(sys, "argv", ["slash_worker", "--session-key", "session-1", "--parent-pid", "424242"])
+ monkeypatch.setattr(sys, "stdin", io.StringIO(""))
+
+ slash_worker.main()
+
+ assert events == [("cli-import",), ("runtime",), ("cli", "session-1")]
+
+
+def test_slash_worker_close_reaps_zombie_and_closes_fds(monkeypatch):
+ """A hung worker is SIGKILLed (its whole group on POSIX), the zombie reaped, all pipes closed — once."""
+ calls = {k: 0 for k in ("terminate", "kill", "wait", "stdin", "stdout", "stderr")}
+ group_signals = []
+
+ class FakeStream:
+ def __init__(self, name):
+ self.name = name
+
+ def close(self):
+ calls[self.name] += 1
+
+ class FakeProc:
+ pid = 424242
+ stdin, stdout, stderr = (FakeStream(n) for n in ("stdin", "stdout", "stderr"))
+
+ def poll(self):
+ return None # always alive -> forces terminate then kill
+
+ def terminate(self):
+ calls["terminate"] += 1
+
+ def kill(self):
+ calls["kill"] += 1
+
+ def wait(self, timeout=None):
+ calls["wait"] += 1
+ raise subprocess.TimeoutExpired(cmd="x", timeout=timeout)
+
+ monkeypatch.setattr(server.os, "killpg", lambda pgid, sig: group_signals.append((pgid, sig)), raising=False)
+ monkeypatch.setattr(server, "release_process_groups", lambda pgids: pytest.fail("a live group was released"))
+ worker = object.__new__(server._SlashWorker)
+ worker.proc = FakeProc()
+
+ worker.close()
+ worker.close() # idempotent
+
+ if os.name == "posix":
+ # TERM then KILL to the worker's group; the trailing 0 is the existence probe that keeps a
+ # group with survivors registered with the owner-death supervisor.
+ assert group_signals == [(424242, signal.SIGTERM), (424242, signal.SIGKILL), (424242, 0)]
+ else:
+ assert calls["terminate"] == calls["kill"] == 1
+ assert calls["wait"] >= 2 # reaped after both terminate and kill
+ assert calls["stdin"] == calls["stdout"] == calls["stderr"] == 1
diff --git a/tests/tui_gateway/test_slash_worker_provider.py b/tests/tui_gateway/test_slash_worker_provider.py
index f56a7331f40..f76b2557215 100644
--- a/tests/tui_gateway/test_slash_worker_provider.py
+++ b/tests/tui_gateway/test_slash_worker_provider.py
@@ -31,7 +31,7 @@ def _spawn(monkeypatch, **worker_kwargs) -> list[str]:
captured["argv"] = list(argv)
return _Popen(argv, **kw)
- monkeypatch.setattr(server.subprocess, "Popen", _popen)
+ monkeypatch.setattr(server, "spawn_owned", lambda argv, purpose, **kw: _popen(argv, **kw))
server._SlashWorker("sess", "default", **worker_kwargs)
return captured["argv"]
@@ -100,9 +100,8 @@ def test_slash_worker_child_forwards_provider_to_hermes_cli(monkeypatch):
def __init__(self, **kwargs):
built.update(kwargs)
- # main() resolves HermesCLI from ``cli`` only after arming the watchdog, so patch it there.
+ # main() resolves HermesCLI from ``cli`` at call time, so patch it there.
monkeypatch.setitem(_sys.modules, "cli", types.SimpleNamespace(HermesCLI=_FakeCLI))
- monkeypatch.setattr(sw, "_start_parent_death_watchdog", lambda *a: None)
monkeypatch.setattr(sw, "_prepare_slash_worker_runtime", lambda: None)
monkeypatch.setattr(sw.sys, "stdin", type("S", (), {"readline": lambda self: ""})())
monkeypatch.setattr(_sys, "argv", ["slash_worker", "--session-key", "s", "--model", "m",
diff --git a/tests/tui_gateway/test_slash_worker_watchdog.py b/tests/tui_gateway/test_slash_worker_watchdog.py
deleted file mode 100644
index 3f0aede0094..00000000000
--- a/tests/tui_gateway/test_slash_worker_watchdog.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import importlib.abc
-import importlib.util
-import io
-import subprocess
-import sys
-from pathlib import Path
-
-from tui_gateway import slash_worker
-
-
-def test_module_import_does_not_load_cli_before_watchdog_can_start():
- subprocess.run(
- [
- sys.executable,
- "-c",
- "import sys; assert 'cli' not in sys.modules; "
- "import tui_gateway.slash_worker; assert 'cli' not in sys.modules",
- ],
- cwd=Path(__file__).resolve().parents[2],
- check=True,
- timeout=10,
- )
-
-
-def test_is_orphaned_true_when_ppid_changes():
- # Our parent went away and we were reparented to a subreaper/init.
- assert slash_worker._is_orphaned(1234, getppid=lambda: 999999) is True
-
-
-def test_is_orphaned_false_when_direct_parent_is_unchanged():
- original_ppid = 1234
- assert slash_worker._is_orphaned(original_ppid, getppid=lambda: original_ppid) is False
-
-
-def test_main_arms_watchdog_then_imports_cli_before_runtime_prep(monkeypatch):
- parent_pid = 424242
- events = []
-
- class FakeHermesCLI:
- def __init__(self, **kwargs):
- events.append(("cli", kwargs["resume"]))
-
- class FakeCliFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader):
- # Record when ``cli`` is actually imported, not just when HermesCLI is built:
- # importing cli loads ~/.hermes/.env, which MCP discovery in runtime prep needs.
- def find_spec(self, name, path=None, target=None):
- return importlib.util.spec_from_loader(name, self) if name == "cli" else None
-
- def create_module(self, spec):
- return None
-
- def exec_module(self, module):
- events.append(("cli-import",))
- module.HermesCLI = FakeHermesCLI
-
- # setitem first so teardown also removes the stub module the import below inserts.
- monkeypatch.setitem(sys.modules, "cli", None)
- monkeypatch.delitem(sys.modules, "cli")
- monkeypatch.setattr(sys, "meta_path", [FakeCliFinder(), *sys.meta_path])
- monkeypatch.setattr(
- slash_worker,
- "_start_parent_death_watchdog",
- lambda pid: events.append(("watchdog", pid)),
- )
- monkeypatch.setattr(
- slash_worker,
- "_prepare_slash_worker_runtime",
- lambda: events.append(("runtime",)),
- )
-
- monkeypatch.setattr(
- sys,
- "argv",
- [
- "slash_worker",
- "--session-key",
- "session-1",
- "--parent-pid",
- str(parent_pid),
- ],
- )
- monkeypatch.setattr(sys, "stdin", io.StringIO(""))
-
- slash_worker.main()
-
- assert events == [
- ("watchdog", slash_worker._watchdog_parent(parent_pid, is_windows=sys.platform == "win32")),
- ("cli-import",),
- ("runtime",),
- ("cli", "session-1"),
- ]
-
-
-def test_watchdog_parent_uses_spawn_pid_off_windows():
- def unexpected_getppid():
- raise AssertionError("spawn parent PID must come from argv")
-
- assert slash_worker._watchdog_parent(424242, is_windows=False, getppid=unexpected_getppid) == 424242
-
-
-def test_watchdog_parent_falls_back_to_observed_parent_without_spawn_pid():
- assert slash_worker._watchdog_parent(0, is_windows=False, getppid=lambda: 777) == 777
-
-
-def test_watchdog_parent_keeps_observed_parent_on_windows():
- # A venv python.exe redirector sits between the gateway and this interpreter on
- # Windows, so the gateway PID never equals os.getppid() there; arming on it
- # would make the watchdog kill a healthy worker on its first poll.
- assert slash_worker._watchdog_parent(424242, is_windows=True, getppid=lambda: 777) == 777
diff --git a/tests/tui_gateway/test_tui_gateway_server.py b/tests/tui_gateway/test_tui_gateway_server.py
index 46bd163dc45..2ce61623134 100644
--- a/tests/tui_gateway/test_tui_gateway_server.py
+++ b/tests/tui_gateway/test_tui_gateway_server.py
@@ -18769,43 +18769,6 @@ def test_sniff_image_ext_magic_and_filename():
assert server._sniff_image_ext(b"\x89PNG", "photo.jpeg") == ".jpeg"
-def test_slash_worker_close_reaps_zombie_and_closes_fds():
- """A hung worker is SIGKILLed, the zombie reaped, all pipes closed — once."""
- calls = {k: 0 for k in ("terminate", "kill", "wait", "stdin", "stdout", "stderr")}
-
- class FakeStream:
- def __init__(self, name):
- self.name = name
-
- def close(self):
- calls[self.name] += 1
-
- class FakeProc:
- stdin, stdout, stderr = (FakeStream(n) for n in ("stdin", "stdout", "stderr"))
-
- def poll(self):
- return None # always alive -> forces terminate then kill
-
- def terminate(self):
- calls["terminate"] += 1
-
- def kill(self):
- calls["kill"] += 1
-
- def wait(self, timeout=None):
- calls["wait"] += 1
- raise subprocess.TimeoutExpired(cmd="x", timeout=timeout)
-
- worker = object.__new__(server._SlashWorker)
- worker.proc = FakeProc()
-
- worker.close()
- worker.close() # idempotent
-
- assert calls["terminate"] == 1
- assert calls["kill"] == 1
- assert calls["wait"] >= 2 # reaped after both terminate and kill
- assert calls["stdin"] == calls["stdout"] == calls["stderr"] == 1
def test_close_session_by_id_is_idempotent_and_full(monkeypatch):
diff --git a/tui_gateway/server.py b/tui_gateway/server.py
index 01f364ebbc7..fc6e6cafcf7 100644
--- a/tui_gateway/server.py
+++ b/tui_gateway/server.py
@@ -12,6 +12,7 @@ import logging
import os
import queue
import re
+import signal
import subprocess
import sys
import threading
@@ -30,6 +31,7 @@ from hermes_constants import (
from hermes_cli import config_backend as _config_backend
from hermes_cli.config_backend import Changes, ConfigBackendUnavailable, config_exists, supports_file_tooling
from hermes_cli.env_loader import load_hermes_dotenv
+from hermes_cli.process_identity import release_process_groups, spawn_owned
from tui_gateway.pet_sheet import _clone_pet_payload, _pet_row_frame_counts, _pet_sheet_revision, _pet_state_rows
from utils import is_truthy_value
from hermes_state_ids import new_session_id
@@ -266,8 +268,7 @@ class _SlashWorker:
# ``--provider`` pins the child to the parent agent's virtual provider: without it the
# worker re-resolves provider from config, so a MoA session (provider=moa, model=<preset>)
# dispatched its preset NAME to the configured real provider and 402/503'd (#57283).
- argv = [sys.executable, "-P", "-m", "tui_gateway.slash_worker", "--session-key", session_key,
- "--parent-pid", str(os.getpid())] \
+ argv = [sys.executable, "-P", "-m", "tui_gateway.slash_worker", "--session-key", session_key] \
+ (["--model", model] if model else []) \
+ (["--provider", provider] if provider else [])
self._closed = False
@@ -293,16 +294,19 @@ class _SlashWorker:
env["PYTHONPATH"] = os.pathsep.join(
part for part in (module_root, env.get("PYTHONPATH", "")) if part
)
- # start_new_session: otherwise the worker inherits the gateway's pgid and mcp_tool's orphan
- # sweep, racing the spawn, killpg()s the TUI parent itself. errors="replace": bytes invalid
- # in the system locale (GBK Windows) must not raise UnicodeDecodeError in the drain threads.
+ # spawn_owned: the worker leads its own process group (otherwise it inherits the gateway's
+ # pgid and mcp_tool's orphan sweep, racing the spawn, killpg()s the TUI parent itself), and
+ # that group dies with this process however it dies: SIGKILL/OOM of the gateway used to leave
+ # a quick command's shell, and anything else the worker started in its group, running forever.
+ # errors="replace": bytes invalid in the system locale (GBK Windows) must not raise
+ # UnicodeDecodeError in the drain threads.
# Prepend the Hermes venv bin dir and the user-local bin dir to PATH so slash_worker child processes
# can resolve Hermes-managed CLIs (browser-use, uvx) even when the parent gateway was launched with
# a minimal PATH (e.g. by the Desktop/Dashboard app). See #83845.
- self.proc = subprocess.Popen(
- argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
- encoding="utf-8", errors="replace", bufsize=1, cwd=os.getcwd(), env=env,
- creationflags=windows_hide_flags(), start_new_session=True)
+ self.proc = spawn_owned(
+ argv, purpose="slash-worker", stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", bufsize=1,
+ cwd=os.getcwd(), env=env, creationflags=windows_hide_flags())
threading.Thread(target=self._drain_stdout, daemon=True).start()
threading.Thread(target=self._drain_stderr, daemon=True).start()
@@ -358,24 +362,48 @@ class _SlashWorker:
self._closed = True
proc = self.proc
try:
- if proc.poll() is None:
- proc.terminate()
- try:
- proc.wait(timeout=1)
- except Exception:
- proc.kill()
- with contextlib.suppress(Exception):
- proc.wait(timeout=1) # reap the zombie SIGKILL leaves behind
- except Exception:
- with contextlib.suppress(Exception):
- proc.kill()
- proc.wait(timeout=1)
+ _stop_worker_group(proc)
finally:
for stream in (proc.stdin, proc.stdout, proc.stderr):
with contextlib.suppress(Exception):
stream.close()
+def _stop_worker_group(proc) -> None:
+ """TERM the slash worker, give it 1 s, then KILL; never raises.
+
+ POSIX signals the worker's whole process group (``spawn_owned`` made it the leader): signalling
+ the leader alone left a running quick command's shell, or anything else the worker started in
+ its group, alive after every session close. The group is released from the owner-death
+ supervisor once it is empty; a group that is not stays covered.
+ """
+ if os.name != "posix":
+ for stop in (proc.terminate, proc.kill):
+ if proc.poll() is not None:
+ return
+ with contextlib.suppress(Exception):
+ stop()
+ proc.wait(timeout=1) # also reaps the zombie a kill leaves behind
+ return
+ pgid = proc.pid
+ # Only a leader still alive on entry proves the number is still this worker's group. A worker
+ # that already died leaves its group to the owner-death supervisor rather than risk signalling
+ # a recycled pgid.
+ if proc.poll() is None:
+ for sig in (signal.SIGTERM, signal.SIGKILL):
+ # Members that outlive the leader keep the group (and its number) alive.
+ with contextlib.suppress(OSError):
+ os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only, Windows returned above
+ with contextlib.suppress(Exception):
+ proc.wait(timeout=1)
+ try:
+ os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only; existence probe, signals nothing
+ except ProcessLookupError:
+ release_process_groups([pgid])
+ except OSError:
+ pass # ambiguous: stay covered by the owner-death supervisor
+
+
def _display_cfg() -> dict:
"""``display`` section of the behavioral config, or ``{}`` when absent/malformed."""
display = _load_cfg().get("display")
diff --git a/tui_gateway/slash_worker.py b/tui_gateway/slash_worker.py
index a9f6d03208c..c8f5faa7980 100644
--- a/tui_gateway/slash_worker.py
+++ b/tui_gateway/slash_worker.py
@@ -21,43 +21,16 @@ import json
import logging
import os
import sys
-import threading
-import time
from typing import TYPE_CHECKING
-from tui_gateway._env import env_float
from tui_gateway._stdin_recovery import handle_spurious_eof
if TYPE_CHECKING:
from cli import HermesCLI
-# Env-overridable so the integration test can drive sub-second timing.
-_WATCHDOG_POLL_S = max(0.05, env_float("HERMES_SLASH_WATCHDOG_POLL_S", 2.0))
-_ORPHAN_GRACE_S = max(0.0, env_float("HERMES_SLASH_WATCHDOG_GRACE_S", 5.0))
-_in_flight = threading.Event() # set while a command is executing
logger = logging.getLogger(__name__)
-def _is_orphaned(original_ppid, getppid=os.getppid) -> bool:
- """Return whether this worker no longer has its original POSIX parent."""
- return getppid() != original_ppid
-
-
-def _watchdog_parent(parent_pid: int, *, is_windows: bool, getppid=os.getppid) -> int:
- """Return the PID the parent-death watchdog should treat as our parent.
-
- The gateway passes its PID at spawn so a fast exit cannot make this child
- mistake a subreaper for its original parent before the watchdog starts.
- The watchdog compares the kernel's live PPID against it, so a reused PID
- can never pass for the parent. Windows never reparents, and a venv
- python.exe redirector makes the launcher (not the gateway) our direct
- parent there, so keep the observed PPID or the worker would exit at once.
- """
- if is_windows or not parent_pid:
- return getppid()
- return parent_pid
-
-
def _prepare_slash_worker_runtime() -> None:
"""Start bounded MCP discovery before HermesCLI snapshots tools: each slash_worker child is its
own process — the parent ``hermes serve`` discovery thread does not populate this registry.
@@ -69,19 +42,6 @@ def _prepare_slash_worker_runtime() -> None:
wait_for_mcp_discovery()
-def _start_parent_death_watchdog(original_ppid) -> None:
- def _loop():
- while not _is_orphaned(original_ppid):
- time.sleep(_WATCHDOG_POLL_S)
- deadline = time.monotonic() + _ORPHAN_GRACE_S
- while _in_flight.is_set() and time.monotonic() < deadline:
- time.sleep(0.05) # let an in-flight command finish/flush
- from hermes_cli.observability.shared_metrics_process import flush_before_hard_exit
- flush_before_hard_exit() # os._exit skips the opt-in metrics runtimes' atexit flush (bounded)
- os._exit(0)
- threading.Thread(target=_loop, daemon=True).start()
-
-
def _slash_base(command: str) -> str:
cmd = (command or "").strip()
cmd = cmd.removeprefix("/")
@@ -165,14 +125,13 @@ def main():
p.add_argument("--session-key", required=True)
p.add_argument("--model", default="")
p.add_argument("--provider", default="")
- p.add_argument("--parent-pid", type=int, default=0)
+ # Accepted and ignored: a gateway started before an update still passes it to the updated worker
+ # code on disk. The worker dies with its gateway through the gateway's owner-death supervisor.
+ p.add_argument("--parent-pid", type=int, default=0, help=argparse.SUPPRESS)
args = p.parse_args()
os.environ["HERMES_SESSION_KEY"] = args.session_key
os.environ["HERMES_INTERACTIVE"] = "1"
- _start_parent_death_watchdog(_watchdog_parent(args.parent_pid, is_windows=sys.platform == "win32"))
- # Keep the heavyweight CLI import behind the watchdog (importing it at module
- # load left a reparenting window before main() could snapshot PPID), but ahead
- # of MCP discovery: importing cli loads ~/.hermes/.env and sets HERMES_QUIET,
+ # Import cli ahead of MCP discovery: importing it loads ~/.hermes/.env and sets HERMES_QUIET,
# which MCP ``${VAR}`` interpolation in the runtime prep depends on.
from cli import HermesCLI
_prepare_slash_worker_runtime()
@@ -197,7 +156,6 @@ def main():
line = raw.strip()
if not line:
continue
- _in_flight.set()
rid = None
try:
req = json.loads(line)
@@ -207,7 +165,6 @@ def main():
except Exception as e:
_reply(id=rid, ok=False, error=str(e))
finally:
- _in_flight.clear()
# Workers persist for the TUI session: release allocator pages at the command boundary like
# other long-lived gateway processes (trim_memory's shared cooldown coalesces nearby activity).
try: