The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(lsp): a language server's whole process tree dies with its Hermes owner (orphans_after_30s 1 -> 0)
kshitijk4poor3d24fce4717d
diff --git a/agent/lsp/__init__.py b/agent/lsp/__init__.pyindex 8ceb6b35ef8..8b866b1494f 100644--- a/agent/lsp/__init__.py+++ b/agent/lsp/__init__.py@@ -41,10 +41,11 @@ def get_service() -> Optional[LSPService]:"""Return the lazily created LSP service for the active profile (process-wide singleton when noprofile override is bound), or None when disabled.- Also registers an :mod:`atexit` hook so a clean exit tears down spawned servers:- without it every ``hermes chat`` exit leaks pyright processes for a few seconds- while their stdout buffers drain. (SIGKILL/os._exit skip atexit — fine, the- kernel reaps the stateless servers with their parent.)+ Also registers an :mod:`atexit` hook so a clean exit tears down spawned servers through+ the LSP ``shutdown``/``exit`` handshake: without it every ``hermes chat`` exit leaks+ pyright processes for a few seconds while their stdout buffers drain. SIGKILL/os._exit+ skip atexit; the owner-death supervisor (``LSPClient._spawn`` hands it each server's+ process group) kills whatever is left then."""global _servicefrom hermes_constants import get_hermes_home_override, hermes_home_keydiff --git a/agent/lsp/client.py b/agent/lsp/client.pyindex 7ea05958bc1..77f831ba589 100644--- a/agent/lsp/client.py+++ b/agent/lsp/client.py@@ -20,6 +20,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Setfrom urllib.parse import quote, unquotefrom hermes_cli._subprocess_compat import windows_hide_flags+from hermes_cli.process_identity import own_process_groups, register_child, release_process_groupsfrom agent.lsp.protocol import (ERROR_CONTENT_MODIFIED, ERROR_METHOD_NOT_FOUND, LSPProtocolError, LSPRequestError, classify_message,@@ -274,10 +275,17 @@ class LSPClient:cmd[0], *cmd[1:], limit=_STREAM_LIMIT,stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,env=delegated_child_subprocess_env({**hermes_subprocess_env(), **(self._env or {})}), cwd=self._cwd,- start_new_session=True, creationflags=windows_hide_flags(),+ start_new_session=True, # detached-spawn: ok — the group goes to own_process_groups below+ creationflags=windows_hide_flags(),)except FileNotFoundError as e:raise LSPProtocolError(f"LSP server binary not found: {cmd[0]} ({e})") from e+ # The server itself exits on stdin EOF when we die, but a descendant that does not hold or+ # watch that pipe (a wrapper's background child, a SIGTERM-ignoring worker) outlived a+ # SIGKILLed / OOM-killed Hermes forever. The owner-death supervisor kills the whole group+ # the moment this process dies, by any cause. No-op on Windows (job object instead).+ own_process_groups([self._proc.pid])+ register_child(self._proc.pid, "lsp-server")# stderr must be drained or the pipe buffer fills and the server hangs.self._stderr_task = asyncio.create_task(self._drain_stderr())self._reader_task = asyncio.create_task(self._reader_loop())@@ -441,6 +449,7 @@ class LSPClient:if proc.returncode is not None:if self._exit_code is None:self._exit_code = proc.returncode+ _release_group_if_empty(proc.pid)returntry:# ``shutdown`` has already given the protocol a grace period. Hard-kill@@ -458,6 +467,7 @@ class LSPClient:passif self._exit_code is None and proc.returncode is not None:self._exit_code = proc.returncode+ _release_group_if_empty(proc.pid)# ---- request / notification plumbing ----@@ -803,6 +813,19 @@ class LSPClient:return _dedupe(push, pull)+def _release_group_if_empty(pgid: int) -> None:+ """Hand the server's process group back from the owner-death supervisor once nothing in it is+ left. A member that survived teardown stays covered, so it still dies with this process."""+ if os.name != "posix":+ return+ try:+ os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only, returned above+ except ProcessLookupError:+ release_process_groups([pgid])+ except OSError:+ pass # exists but not ours to probe: keep covering it++def _dedupe(*lists: list[dict[str, Any]]) -> list[dict[str, Any]]:seen: set[str] = set()out: list[dict[str, Any]] = []diff --git a/tests/agent/lsp/test_lsp_owned_server.py b/tests/agent/lsp/test_lsp_owned_server.pynew file mode 100644index 00000000000..9158dedac05--- /dev/null+++ b/tests/agent/lsp/test_lsp_owned_server.py@@ -0,0 +1,163 @@+"""LSP servers die with the Hermes process that started them, however it dies.++``LSPClient`` starts each language server in its own session/process group so a killpg aimed at+the server can never reach Hermes. That also took the server's descendants out of Hermes's kill+domain: the server itself exits on stdin EOF when Hermes dies, but a descendant that neither holds+nor watches that pipe (a wrapper's background child, a SIGTERM-ignoring worker) outlived a+SIGKILLed / OOM-killed Hermes forever. ``_spawn`` now hands the group to the per-process+owner-death supervisor; teardown hands it back once the group is empty.+"""+from __future__ import annotations++import os+import subprocess+import sys+import textwrap+from pathlib import Path++import psutil+import pytest++from agent.lsp import client as lsp_client+from agent.lsp.client import LSPClient+from hermes_cli import process_identity+from tests.waiting import wait_until++pytestmark = pytest.mark.platforms("posix")++MOCK_SERVER = str(Path(__file__).parent / "_mock_lsp_server.py")+REPO = str(Path(__file__).resolve().parents[3])+++def _alive(pid: int) -> bool:+ try:+ return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE+ except psutil.NoSuchProcess:+ return False+++def _client(workspace: Path, script: str, **extra_env: str) -> LSPClient:+ env = {"MOCK_LSP_SCRIPT": script, "PYTHONPATH": os.environ.get("PYTHONPATH", ""), **extra_env}+ return LSPClient(server_id=f"mock-{script}", workspace_root=str(workspace),+ command=[sys.executable, MOCK_SERVER], env=env, cwd=str(workspace))+++def recorded(monkeypatch):+ calls: list = []+ monkeypatch.setattr(lsp_client, "own_process_groups", lambda pgids: calls.append(("own", list(pgids))))+ monkeypatch.setattr(lsp_client, "release_process_groups", lambda pgids: calls.append(("release", list(pgids))))+ monkeypatch.setattr(lsp_client, "register_child", lambda pid, purpose: calls.append(("ledger", pid, purpose)))+ return calls+++async def test_the_server_group_is_owned_on_spawn_and_released_on_shutdown(tmp_path, recorded):+ client = _client(tmp_path, "clean")+ await client.start()+ pid = client._proc.pid+ try:+ assert os.getpgid(pid) == pid # its own group: a killpg of it can never reach Hermes+ assert recorded == [("own", [pid]), ("ledger", pid, "lsp-server")]+ finally:+ await client.shutdown()+ assert recorded[-1] == ("release", [pid])++[email protected]_system_guard_bypass # the child reparents to init once its leader is killed+async def test_a_surviving_group_member_stays_owned(tmp_path, recorded, monkeypatch):+ """Teardown that could not empty the group must not drop coverage: whatever survived still+ has to die with Hermes."""+ monkeypatch.setattr("agent.deadline.kill_process_tree", lambda pid, **kw: True) # kills nothing+ child_pid_file = tmp_path / "child.pid"+ client = _client(tmp_path, "slow_tree", MOCK_LSP_CHILD_PID=str(child_pid_file))+ await client.start()+ leader = client._proc.pid+ child = int(wait_until(lambda: child_pid_file.exists() and child_pid_file.read_text(encoding="utf-8"),+ timeout=15, what="the server's SIGTERM-ignoring child to start"))+ try:+ monkeypatch.setattr(lsp_client, "SHUTDOWN_GRACE", 0.1)+ client._proc.kill() # the leader goes; its child stays in the group+ await client.shutdown()+ assert not [c for c in recorded if c[0] == "release"]+ finally:+ for pid in (child, leader):+ try:+ os.kill(pid, 9) # windows-footgun: ok — posix-marked test+ except ProcessLookupError:+ pass+++def _owner(tmp_path: Path, script: str) -> tuple[subprocess.Popen, int, int]:+ """A real Hermes-like owner: starts an LSPClient on the mock server's ``slow_tree`` script (a+ leader plus a SIGTERM-ignoring child that never touches the stdio pipes) and prints both pids."""+ child_pid_file = tmp_path / "child.pid"+ ws = tmp_path / "ws"+ ws.mkdir()+ src = textwrap.dedent(f"""+ import asyncio, os, sys, time+ from agent.lsp.client import LSPClient+ async def main():+ c = LSPClient(server_id="mock", workspace_root={str(ws)!r}, command=[sys.executable, {MOCK_SERVER!r}],+ env={{"MOCK_LSP_SCRIPT": {script!r}, "MOCK_LSP_CHILD_PID": {str(child_pid_file)!r}}})+ await c.start()+ while not os.path.exists({str(child_pid_file)!r}):+ await asyncio.sleep(0.05)+ print(c._proc.pid, open({str(child_pid_file)!r}, encoding="utf-8").read(), flush=True)+ await asyncio.sleep(300)+ asyncio.run(main())+ """)+ home = tmp_path / "home"+ home.mkdir()+ env = dict(os.environ, HERMES_HOME=str(home), HOME=str(home), PYTHONPATH=REPO)+ owner = subprocess.Popen([sys.executable, "-c", src], env=env, cwd=REPO, stdin=subprocess.DEVNULL,+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)+ line = owner.stdout.readline().split()+ if len(line) != 2:+ owner.kill()+ pytest.fail(f"owner did not start the server: {line!r} {owner.communicate(timeout=10)[1][-2000:]}")+ return owner, int(line[0]), int(line[1])+++def _cleanup(owner: subprocess.Popen, *pids: int) -> None:+ for pid in pids:+ try:+ os.kill(pid, 9) # windows-footgun: ok — posix-marked test+ except ProcessLookupError:+ pass+ if owner.poll() is None:+ owner.kill()+ owner.communicate(timeout=10)+ process_identity._supervised_pgids.clear()+++# The server tree reparents to init once its owner dies, so cleanup leaves this test's subtree.[email protected]_system_guard_bypass+def test_a_sigkilled_owner_takes_its_whole_server_tree_with_it(tmp_path):+ """End to end through the real supervisor: the owner dies by SIGKILL (no Python cleanup runs).+ The leader exits on stdin EOF by itself; its SIGTERM-ignoring child, which holds no pipe, used+ to survive forever (measured on main: alive 30 s later, 3/3 trials)."""+ owner, leader, child = _owner(tmp_path, "slow_tree")+ try:+ owner.kill()+ owner.wait(timeout=10)+ wait_until(lambda: not _alive(leader) and not _alive(child), timeout=15,+ what="the LSP server and its descendant to die with the kill -9'd Hermes process")+ finally:+ _cleanup(owner, leader, child)++[email protected]_system_guard_bypass+def test_a_live_owner_keeps_its_server_tree(tmp_path):+ """Control: while the owner lives, the supervisor touches nothing."""+ owner, leader, child = _owner(tmp_path, "slow_tree")+ try:+ supervisor = [p for p in psutil.Process(owner.pid).children() if "process_supervisor.sh" in " ".join(p.cmdline())]+ assert supervisor, "the owner should have started its owner-death supervisor"+ # Give a wrongly-firing supervisor its full TERM -> 3 s -> KILL window to act.+ with pytest.raises(AssertionError):+ wait_until(lambda: not _alive(leader) or not _alive(child), timeout=5, what="(control) a server death")+ assert _alive(leader) and _alive(child) and owner.poll() is None+ finally:+ _cleanup(owner, leader, child)diff --git a/tests/test_windows_subprocess_no_window_flags.py b/tests/test_windows_subprocess_no_window_flags.pyindex eed145aef20..9584630399a 100644--- a/tests/test_windows_subprocess_no_window_flags.py+++ b/tests/test_windows_subprocess_no_window_flags.py@@ -277,6 +277,7 @@ def test_lsp_client_spawn_hides_console_window(monkeypatch):captured = []class _FakeProc:+ pid = 424242stdin = Nonestdout = Nonestderr = None@@ -286,6 +287,9 @@ def test_lsp_client_spawn_hides_console_window(monkeypatch):return _FakeProc()monkeypatch.setattr(lsp_client, "windows_hide_flags", lambda: _CREATE_NO_WINDOW)+ # No real child: keep the fake pid away from the owner-death supervisor and the spawn ledger.+ monkeypatch.setattr(lsp_client, "own_process_groups", lambda pgids: None)+ monkeypatch.setattr(lsp_client, "register_child", lambda pid, purpose: None)monkeypatch.setattr(lsp_client.asyncio, "create_subprocess_exec", fake_exec)