Skip to content

The agent that grows with you

fix(processes): a background job dies with the Hermes process that started it (orphans_after_10s 3 -> 0); persist_on_release jobs still outlive it

kshitijk4poor755a280ff045
diff --git a/tests/tools/test_local_setsid_descendant_sweep.py b/tests/tools/test_local_setsid_descendant_sweep.py
index 49da17d9e9a..392231ce757 100644
--- a/tests/tools/test_local_setsid_descendant_sweep.py
+++ b/tests/tools/test_local_setsid_descendant_sweep.py
@@ -196,7 +196,7 @@ class TestReapUntrackedGroupGuard:
reg = ProcessRegistry()
proc = MagicMock()
proc.pid = 999
- session = SimpleNamespace(systemd_unit="", host_start_time=4242)
+ session = SimpleNamespace(systemd_unit="", _owned_pgid=None, host_start_time=4242)
calls, forbidden = [], []
monkeypatch.setattr(
ProcessRegistry, "_terminate_host_pid",
@@ -223,7 +223,7 @@ class TestReapUntrackedGroupGuard:
try:
real_start = ProcessRegistry._safe_host_start_time(proc.pid)
assert real_start is not None, "no /proc start time on this platform?"
- session = SimpleNamespace(systemd_unit="", host_start_time=real_start)
+ session = SimpleNamespace(systemd_unit="", _owned_pgid=None, host_start_time=real_start)
ProcessRegistry()._reap_untracked(session, proc)
@@ -248,7 +248,7 @@ class TestReapUntrackedGroupGuard:
real_start = ProcessRegistry._safe_host_start_time(proc.pid)
assert real_start is not None, "no /proc start time on this platform?"
assert os.getpgid(proc.pid) == proc.pid # leads its group: old arm fired here
- session = SimpleNamespace(systemd_unit="", host_start_time=real_start + 1)
+ session = SimpleNamespace(systemd_unit="", _owned_pgid=None, host_start_time=real_start + 1)
# A stub Popen keeps the wait tail from parking 5s on the still-live child.
stub = SimpleNamespace(pid=proc.pid, wait=lambda timeout=None: None)
diff --git a/tests/tools/test_terminal_owned_jobs.py b/tests/tools/test_terminal_owned_jobs.py
index 87b7a127e65..36c0b77d6af 100644
--- a/tests/tools/test_terminal_owned_jobs.py
+++ b/tests/tools/test_terminal_owned_jobs.py
@@ -1,7 +1,9 @@
-"""POSIX: terminal commands die with the Hermes process that started them.
+"""POSIX: terminal commands and background jobs die with the Hermes process that started them.
-The owners are real host processes using the real ``LocalEnvironment.execute``. One owner is SIGKILLed,
-so no Python cleanup of ours runs; the other is the live-owner control.
+The owners are real host processes using the real ``ProcessRegistry.spawn_local`` and
+``LocalEnvironment.execute``. One owner is SIGKILLed, so no Python cleanup of ours runs; the other is the
+live-owner control. persist_on_release jobs (#41225) and the jobs of a checkpoint-recovering host must
+outlive the owner.
"""
import json
@@ -21,8 +23,19 @@ _OWNER = r'''
import json, os, sys, threading, time
from pathlib import Path
root, role = Path(sys.argv[1]), sys.argv[2]
+from tools.process_registry import process_registry
from tools.environments.local import LocalEnvironment
+if role.startswith("recovering"):
+ process_registry.recover_from_checkpoint()
ids = {}
+for name, kwargs in (("job", {}), ("pty_job", {"use_pty": True}), ("persisted_job", {"persist_on_release": True})):
+ marker = root / f"{role}.{name}"
+ # A job that ignores SIGHUP (nohup'd servers): a PTY hangup alone does not stop it.
+ s = process_registry.spawn_local(f"trap '' HUP; sleep 120 & echo $! > {marker}; wait", cwd=str(root), **kwargs)
+ wait_until = time.time() + 30
+ while not marker.exists() and time.time() < wait_until:
+ time.sleep(0.05)
+ ids[name] = [s.pid, int(marker.read_text().strip())]
env = LocalEnvironment(cwd=str(root), timeout=600)
fg = root / f"{role}.fg"
threading.Thread(target=env.execute, args=(f"sleep 120 & echo $! > {fg}; wait",), daemon=True).start()
@@ -78,10 +91,10 @@ def _cleanup(owners, log, ids):
log.close()
-# Commands reparent to init once their owner dies, so cleaning them up leaves this test's subtree; real
+# Jobs reparent to init once their owner dies, so cleaning them up leaves this test's subtree; real
# signal delivery to real orphans is the point of the test.
@pytest.mark.live_system_guard_bypass
-def test_owner_sigkill_takes_its_running_command_not_a_live_owners(tmp_path):
+def test_owner_sigkill_takes_its_jobs_and_commands_not_a_live_owners(tmp_path):
owners, log = _run_owners(tmp_path, ("control", "victim"))
ids = {}
try:
@@ -89,28 +102,108 @@ def test_owner_sigkill_takes_its_running_command_not_a_live_owners(tmp_path):
owners["victim"].kill() # SIGKILL: no atexit, no finally, nothing of ours runs
owners["victim"].wait(timeout=10)
- wait_until(lambda: not any(_alive(i) for i in ids["victim"]["foreground"]), timeout=15,
- what="the dead owner's foreground command to exit")
- assert all(_alive(i) for i in ids["control"]["foreground"]), (
- "a live owner's command was killed", ids["control"])
+ owned = {name: job for name, job in ids["victim"].items() if name != "persisted_job"}
+ wait_until(lambda: not any(_alive(i) for job in owned.values() for i in job), timeout=15,
+ what="the dead owner's background jobs and foreground command to exit")
+ assert all(_alive(i) for i in ids["victim"]["persisted_job"]), (
+ "a persist_on_release job died with its owner", ids["victim"]["persisted_job"])
+ assert all(_alive(i) for job in ids["control"].values() for i in job), (
+ "a live owner's jobs were killed", ids["control"])
+ finally:
+ _cleanup(owners, log, ids)
+
+
[email protected]_system_guard_bypass
+def test_a_checkpoint_recovering_hosts_jobs_outlive_its_crash(tmp_path):
+ owners, log = _run_owners(tmp_path, ("recovering",))
+ ids = {}
+ try:
+ ids = {"recovering": _read(tmp_path / "recovering.json")}
+ owners["recovering"].kill()
+ owners["recovering"].wait(timeout=10)
+ # The foreground command is not a job: it dies even here.
+ wait_until(lambda: not any(_alive(i) for i in ids["recovering"]["foreground"]), timeout=15,
+ what="the dead host's foreground command to exit")
+ jobs = [i for name, job in ids["recovering"].items() if name != "foreground" for i in job]
+ assert all(_alive(i) for i in jobs), (
+ "a recovering host's background job died with it; the next host could not re-adopt it", jobs)
finally:
_cleanup(owners, log, ids)
def _record_calls(monkeypatch):
+ from tools import process_registry_ownership as registry_mod
from tools.environments import local as local_mod
calls = []
- monkeypatch.setattr(local_mod, "own_process_groups", lambda pgids: calls.append(("own", list(pgids))))
- monkeypatch.setattr(local_mod, "release_process_groups", lambda pgids: calls.append(("release", list(pgids))))
+ for mod in (registry_mod, local_mod):
+ monkeypatch.setattr(mod, "own_process_groups", lambda pgids: calls.append(("own", list(pgids))))
+ monkeypatch.setattr(mod, "release_process_groups", lambda pgids: calls.append(("release", list(pgids))))
+ monkeypatch.setattr("hermes_cli.process_identity.own_process_groups",
+ lambda pgids: calls.append(("own", list(pgids))))
+ monkeypatch.setattr("hermes_cli.process_identity.register_child", lambda pid, purpose: None)
return calls
-def test_a_returned_command_releases_its_group(monkeypatch, tmp_path):
+def test_a_finished_job_and_a_returned_command_release_their_group(monkeypatch, tmp_path):
from tools.environments.local import LocalEnvironment
+ from tools.process_registry import ProcessRegistry
calls = _record_calls(monkeypatch)
+ registry = ProcessRegistry()
+ session = registry.spawn_local("exit 0", cwd=str(tmp_path))
+ wait_until(lambda: session.exited and ("release", [session.pid]) in calls, timeout=15,
+ what="the finished job's release")
+ assert calls[0] == ("own", [session.pid])
+
+ calls.clear()
env = LocalEnvironment(cwd=str(tmp_path), timeout=30)
calls.clear()
assert env.execute("echo hi")["returncode"] == 0
assert [verb for verb, _ in calls] == ["own", "release"] and calls[0][1] == calls[1][1], calls
+
+
+def test_a_command_yielded_to_background_keeps_its_group_until_the_job_ends(monkeypatch, tmp_path):
+ import threading
+
+ from tools.environments.local import LocalEnvironment
+ from tools.interrupt import request_yield
+ from tools.process_registry import ProcessRegistry
+
+ calls = _record_calls(monkeypatch)
+ env = LocalEnvironment(cwd=str(tmp_path), timeout=60)
+ registry = ProcessRegistry()
+ gate, started = tmp_path / "gate", tmp_path / "started"
+ adopted = []
+
+ def handler(proc, output_so_far):
+ adopted.append(registry.adopt_local(proc, command="wait for gate", cwd=str(tmp_path)))
+ return {"output": output_so_far, "returncode": None}
+
+ result = {}
+ worker = threading.Thread(target=lambda: result.update(env.execute(
+ f"touch {started}; while [ ! -e {gate} ]; do sleep 0.05; done", yield_handler=handler)))
+ calls.clear()
+ worker.start()
+ wait_until(started.exists, timeout=15, what="the foreground command to start")
+ request_yield(worker.ident)
+ worker.join(timeout=15)
+ assert result.get("returncode") is None and adopted, result
+ session = adopted[0]
+ assert calls == [("own", [session.pid])], "the yielding foreground wait released the job's group"
+ assert session._owned_pgid == session.pid
+ gate.touch()
+ wait_until(lambda: ("release", [session.pid]) in calls, timeout=15, what="the adopted job's release")
+ assert calls.count(("release", [session.pid])) == 1, calls
+
+
+def test_unowned_jobs_are_never_handed_to_the_supervisor(monkeypatch, tmp_path):
+ from tools.process_registry import ProcessRegistry
+
+ calls = _record_calls(monkeypatch)
+ registry = ProcessRegistry()
+ persisted = registry.spawn_local("exit 0", cwd=str(tmp_path), persist_on_release=True)
+ registry._jobs_outlive_host = True
+ recovered_host_job = registry.spawn_local("exit 0", cwd=str(tmp_path))
+ wait_until(lambda: persisted.exited and recovered_host_job.exited, timeout=15, what="both jobs to exit")
+ assert calls == [], calls
diff --git a/tools/environments/local.py b/tools/environments/local.py
index 47b6ee2139c..4d0bd5ed215 100644
--- a/tools/environments/local.py
+++ b/tools/environments/local.py
@@ -1171,7 +1171,8 @@ class LocalEnvironment(BaseEnvironment):
def _wait_for_process(self, proc, *args, **kwargs) -> dict:
"""Base wait, then hand the command's group back from the owner-death supervisor: once the
command returned, what it left running in the background (`server &`) is not ours to kill,
- exactly as on a graceful exit."""
+ exactly as on a graceful exit. A command that yielded to background took the group with it
+ (``ProcessRegistry.adopt_local`` clears ``_hermes_owned_pgid``)."""
try:
return super()._wait_for_process(proc, *args, **kwargs)
finally:
diff --git a/tools/process_registry.py b/tools/process_registry.py
index 7f55acd6b87..52defc75c7a 100644
--- a/tools/process_registry.py
+++ b/tools/process_registry.py
@@ -36,6 +36,7 @@ from tools.process_registry_notifications import (
format_process_notification, heartbeat_output_tail, is_child_process_notification, should_surface_notification)
from tools.process_registry_checkpoint import ProcessCheckpointMixin
from tools.process_registry_output_log import ProcessOutputLogMixin, open_output_log, record_exit_command, worker_output_log
+from tools.process_registry_ownership import ProcessOwnershipMixin
from tools.process_registry_termination import ProcessTerminationMixin
from tools.process_registry_results import load_completed_results, save_completed_result
from tools.process_registry_env_log import log_delta_command
@@ -609,6 +610,7 @@ class ProcessSession:
_reader_finish_requested: threading.Event = field(default_factory=threading.Event, repr=False)
_reader_selectable: bool = field(default=False, repr=False)
_pty: Any = field(default=None, repr=False) # ptyprocess handle (use_pty=True)
+ _owned_pgid: Optional[int] = field(default=None, repr=False) # see ProcessOwnershipMixin
def __post_init__(self):
# A session built without an explicit owner is owned by its own task, so ownership checks compare
@@ -672,7 +674,7 @@ _WSL_CHAIN_NOTE = (
)
-class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOutputLogMixin):
+class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOutputLogMixin, ProcessOwnershipMixin):
"""In-memory registry of running and finished background processes.
Thread-safe: accessed from executor threads (terminal_tool, process handlers),
the gateway asyncio loop (watchers, reset checks) and the cleanup thread."""
@@ -1167,6 +1169,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu
pty_env.setdefault("GIT_PAGER", "cat")
pty_env.setdefault("PAGER", "cat")
pty_proc = _PtyProcessCls.spawn(pty_argv, cwd=session.cwd, env=pty_env, dimensions=(30, 120))
+ self._own_pty_job(session, pty_proc)
session.pid = pty_proc.pid
session.host_start_time = self._safe_host_start_time(session.pid)
session._pty = pty_proc
@@ -1222,10 +1225,10 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu
# the scope attaches to the invoked process, not the spawning session.
output = open_output_log(log) if log else subprocess.PIPE
try:
- proc = subprocess.Popen(
- spawn_argv, text=True, cwd=session.cwd, env=spawn_env, encoding="utf-8",
- errors="replace", stdout=output, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL,
- start_new_session=True, **_popen_kwargs)
+ proc = self._spawn_job(
+ session, spawn_argv, persist_on_release, text=True, cwd=session.cwd, env=spawn_env,
+ encoding="utf-8", errors="replace", stdout=output, stderr=subprocess.STDOUT,
+ stdin=subprocess.DEVNULL, **_popen_kwargs)
finally:
if log:
os.close(output)
@@ -1239,21 +1242,6 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu
raise
return session
- def _reap_untracked(self, session: ProcessSession, proc: subprocess.Popen) -> None:
- """Post-Popen setup failed: kill the orphaned subprocess (and any setsid
- descendants) so nothing leaks untracked."""
- with suppress(Exception):
- if session.systemd_unit:
- # Scope teardown is the authoritative cleanup for the worker cgroup;
- # the systemd-run wrapper PID is terminated below as fallback.
- _stop_systemd_unit(session.systemd_unit)
- # _terminate_host_pid revalidates the kernel start time captured at
- # spawn before signalling: pgid == pid would only prove the pid leads a
- # group, and a reaped child can be recycled onto an unrelated leader.
- self._terminate_host_pid(proc.pid, session.host_start_time)
- with suppress(Exception):
- proc.wait(timeout=5)
-
def adopt_local(
self, proc: subprocess.Popen, *, command: str, cwd: Optional[str], task_id: str = "",
session_key: str = "", owner_task_id: str = "", output_so_far: str = "",
@@ -1268,6 +1256,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu
session.pid = proc.pid
session.host_start_time = self._safe_host_start_time(session.pid)
session.notify_on_complete = notify_on_complete
+ self._take_handoff_group(session, proc)
if output_so_far:
session.append_output(output_so_far)
self._track_started(session, self._reader_loop, f"proc-reader-{session.id}")
@@ -1567,6 +1556,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu
save_completed_result(session)
self._running.pop(session.id)
self._finished[session.id] = session
+ self._release_job_group(session)
# Release the retained Popen/PTY handles now: otherwise every
# finished-but-unpruned session keeps its stdout pipe (or PTY master)
# FD open until FINISHED_TTL_SECONDS elapses, and heavy background
diff --git a/tools/process_registry_checkpoint.py b/tools/process_registry_checkpoint.py
index c05ab07a5ea..176f0c3daba 100644
--- a/tools/process_registry_checkpoint.py
+++ b/tools/process_registry_checkpoint.py
@@ -78,6 +78,9 @@ class ProcessCheckpointMixin:
)
from tools.process_registry_output_log import adoptable_log
+ # This host adopts what a crashed predecessor left running, so the jobs it starts must
+ # survive its own crash for the next one: they stay out of the owner-death supervisor.
+ self._jobs_outlive_host = True
checkpoint_path = _checkpoint_path()
if not checkpoint_path.exists():
return 0
diff --git a/tools/process_registry_ownership.py b/tools/process_registry_ownership.py
new file mode 100644
index 00000000000..22341ce825d
--- /dev/null
+++ b/tools/process_registry_ownership.py
@@ -0,0 +1,82 @@
+"""Which local background jobs die with their owner, for ProcessRegistry (process_identity layer 4)."""
+
+import subprocess
+import threading
+from contextlib import suppress
+from typing import Any
+
+from hermes_cli.process_identity import own_process_groups, release_process_groups, spawn_owned
+
+
+class ProcessOwnershipMixin:
+ """A local job's process group goes to the owner-death supervisor while the job's own process runs,
+ so it dies with this process however this process dies (SIGKILL, OOM, crash): the agent-lifecycle
+ sweeps only run while we are alive, and a killed CLI/TUI/worker left every job running on init.
+ ``ProcessSession._owned_pgid`` is that group, cleared by the one release so a recycled pgid is
+ never unregistered on someone else's behalf. The subclass supplies ``_terminate_host_pid``."""
+
+ # Set by recover_from_checkpoint: this host re-adopts the jobs a crashed predecessor left
+ # running, so its own jobs must outlive its crash the same way (see _owns_job).
+ _jobs_outlive_host = False
+ _owned_pgid_lock = threading.Lock()
+
+ def _owns_job(self, persist_on_release: bool) -> bool:
+ """Not owned, on purpose: ``persist_on_release`` jobs, which must outlive the agent (#41225);
+ every job of a gateway, which re-adopts its jobs from the checkpoint after a crash (flagged by
+ recover_from_checkpoint; a supervised gateway also before it gets there); and Windows, where
+ the layer-3 job object covers ownership."""
+ from tools.process_registry import _IS_WINDOWS, _is_supervised_gateway_process
+
+ return not (_IS_WINDOWS or persist_on_release or self._jobs_outlive_host
+ or _is_supervised_gateway_process())
+
+ def _spawn_job(self, session: Any, argv: list[str], persist_on_release: bool, **popen_kwargs) -> subprocess.Popen:
+ """Start a pipe job in its own session, owned unless ``_owns_job`` says it must outlive us."""
+ if self._owns_job(persist_on_release):
+ proc = spawn_owned(argv, purpose="terminal-background", **popen_kwargs)
+ session._owned_pgid = proc.pid
+ return proc
+ # noqa: subprocess-stdin — a forwarder: spawn_local passes stdin=DEVNULL in popen_kwargs
+ return subprocess.Popen(argv, start_new_session=True, **popen_kwargs) # detached-spawn: ok — see _owns_job
+
+ def _own_pty_job(self, session: Any, pty_proc: Any) -> None:
+ """The PTY child leads its own session and group (pty.fork). Its hangup reaches only jobs that
+ keep the default SIGHUP action; `nohup`/trap-HUP servers outlived a killed owner."""
+ if self._owns_job(session.persist_on_release):
+ own_process_groups([pty_proc.pid])
+ session._owned_pgid = pty_proc.pid
+
+ def _take_handoff_group(self, session: Any, proc: subprocess.Popen) -> None:
+ """A foreground command that yields to background hands its supervised group over with the
+ pipe (LocalEnvironment._run_bash): the job keeps it, unless jobs here must outlive us."""
+ owned_pgid, proc._hermes_owned_pgid = getattr(proc, "_hermes_owned_pgid", None), None
+ if owned_pgid is not None and self._owns_job(False):
+ session._owned_pgid = owned_pgid
+ elif owned_pgid is not None:
+ release_process_groups([owned_pgid])
+
+ def _release_job_group(self, session: Any) -> None:
+ """The job's own process exited or was killed: stop covering its group. What it left behind on
+ purpose (`server &`) is no longer this job's, exactly as before."""
+ with self._owned_pgid_lock:
+ owned_pgid, session._owned_pgid = session._owned_pgid, None
+ if owned_pgid is not None:
+ release_process_groups([owned_pgid])
+
+ def _reap_untracked(self, session: Any, proc: subprocess.Popen) -> None:
+ """Post-Popen setup failed: kill the orphaned subprocess (and any setsid
+ descendants) so nothing leaks untracked."""
+ from tools.process_registry import _stop_systemd_unit
+
+ with suppress(Exception):
+ if session.systemd_unit:
+ # Scope teardown is the authoritative cleanup for the worker cgroup;
+ # the systemd-run wrapper PID is terminated below as fallback.
+ _stop_systemd_unit(session.systemd_unit)
+ # _terminate_host_pid revalidates the kernel start time captured at
+ # spawn before signalling: pgid == pid would only prove the pid leads a
+ # group, and a reaped child can be recycled onto an unrelated leader.
+ self._terminate_host_pid(proc.pid, session.host_start_time)
+ with suppress(Exception):
+ proc.wait(timeout=5)
+ self._release_job_group(session)