The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(processes): a background job dies with the Hermes process that started it (orphans_after_10s 3 -> 0); persist_on_release jobs still outlive it
kshitijk4poor755a280ff045
diff --git a/tests/tools/test_local_setsid_descendant_sweep.py b/tests/tools/test_local_setsid_descendant_sweep.pyindex 49da17d9e9a..392231ce757 100644--- a/tests/tools/test_local_setsid_descendant_sweep.py+++ b/tests/tools/test_local_setsid_descendant_sweep.py@@ -196,7 +196,7 @@ class TestReapUntrackedGroupGuard:reg = ProcessRegistry()proc = MagicMock()proc.pid = 999- session = SimpleNamespace(systemd_unit="", host_start_time=4242)+ session = SimpleNamespace(systemd_unit="", _owned_pgid=None, host_start_time=4242)calls, forbidden = [], []monkeypatch.setattr(ProcessRegistry, "_terminate_host_pid",@@ -223,7 +223,7 @@ class TestReapUntrackedGroupGuard:try:real_start = ProcessRegistry._safe_host_start_time(proc.pid)assert real_start is not None, "no /proc start time on this platform?"- session = SimpleNamespace(systemd_unit="", host_start_time=real_start)+ session = SimpleNamespace(systemd_unit="", _owned_pgid=None, host_start_time=real_start)ProcessRegistry()._reap_untracked(session, proc)@@ -248,7 +248,7 @@ class TestReapUntrackedGroupGuard:real_start = ProcessRegistry._safe_host_start_time(proc.pid)assert real_start is not None, "no /proc start time on this platform?"assert os.getpgid(proc.pid) == proc.pid # leads its group: old arm fired here- session = SimpleNamespace(systemd_unit="", host_start_time=real_start + 1)+ session = SimpleNamespace(systemd_unit="", _owned_pgid=None, host_start_time=real_start + 1)# A stub Popen keeps the wait tail from parking 5s on the still-live child.stub = SimpleNamespace(pid=proc.pid, wait=lambda timeout=None: None)diff --git a/tests/tools/test_terminal_owned_jobs.py b/tests/tools/test_terminal_owned_jobs.pyindex 87b7a127e65..36c0b77d6af 100644--- a/tests/tools/test_terminal_owned_jobs.py+++ b/tests/tools/test_terminal_owned_jobs.py@@ -1,7 +1,9 @@-"""POSIX: terminal commands die with the Hermes process that started them.+"""POSIX: terminal commands and background jobs die with the Hermes process that started them.-The owners are real host processes using the real ``LocalEnvironment.execute``. One owner is SIGKILLed,-so no Python cleanup of ours runs; the other is the live-owner control.+The owners are real host processes using the real ``ProcessRegistry.spawn_local`` and+``LocalEnvironment.execute``. One owner is SIGKILLed, so no Python cleanup of ours runs; the other is the+live-owner control. persist_on_release jobs (#41225) and the jobs of a checkpoint-recovering host must+outlive the owner."""import json@@ -21,8 +23,19 @@ _OWNER = r'''import json, os, sys, threading, timefrom pathlib import Pathroot, role = Path(sys.argv[1]), sys.argv[2]+from tools.process_registry import process_registryfrom tools.environments.local import LocalEnvironment+if role.startswith("recovering"):+ process_registry.recover_from_checkpoint()ids = {}+for name, kwargs in (("job", {}), ("pty_job", {"use_pty": True}), ("persisted_job", {"persist_on_release": True})):+ marker = root / f"{role}.{name}"+ # A job that ignores SIGHUP (nohup'd servers): a PTY hangup alone does not stop it.+ s = process_registry.spawn_local(f"trap '' HUP; sleep 120 & echo $! > {marker}; wait", cwd=str(root), **kwargs)+ wait_until = time.time() + 30+ while not marker.exists() and time.time() < wait_until:+ time.sleep(0.05)+ ids[name] = [s.pid, int(marker.read_text().strip())]env = LocalEnvironment(cwd=str(root), timeout=600)fg = root / f"{role}.fg"threading.Thread(target=env.execute, args=(f"sleep 120 & echo $! > {fg}; wait",), daemon=True).start()@@ -78,10 +91,10 @@ def _cleanup(owners, log, ids):log.close()-# Commands reparent to init once their owner dies, so cleaning them up leaves this test's subtree; real+# Jobs reparent to init once their owner dies, so cleaning them up leaves this test's subtree; real# signal delivery to real orphans is the point of the test.@pytest.mark.live_system_guard_bypass-def test_owner_sigkill_takes_its_running_command_not_a_live_owners(tmp_path):+def test_owner_sigkill_takes_its_jobs_and_commands_not_a_live_owners(tmp_path):owners, log = _run_owners(tmp_path, ("control", "victim"))ids = {}try:@@ -89,28 +102,108 @@ def test_owner_sigkill_takes_its_running_command_not_a_live_owners(tmp_path):owners["victim"].kill() # SIGKILL: no atexit, no finally, nothing of ours runsowners["victim"].wait(timeout=10)- wait_until(lambda: not any(_alive(i) for i in ids["victim"]["foreground"]), timeout=15,- what="the dead owner's foreground command to exit")- assert all(_alive(i) for i in ids["control"]["foreground"]), (- "a live owner's command was killed", ids["control"])+ owned = {name: job for name, job in ids["victim"].items() if name != "persisted_job"}+ wait_until(lambda: not any(_alive(i) for job in owned.values() for i in job), timeout=15,+ what="the dead owner's background jobs and foreground command to exit")+ assert all(_alive(i) for i in ids["victim"]["persisted_job"]), (+ "a persist_on_release job died with its owner", ids["victim"]["persisted_job"])+ assert all(_alive(i) for job in ids["control"].values() for i in job), (+ "a live owner's jobs were killed", ids["control"])+ finally:+ _cleanup(owners, log, ids)++[email protected]_system_guard_bypass+def test_a_checkpoint_recovering_hosts_jobs_outlive_its_crash(tmp_path):+ owners, log = _run_owners(tmp_path, ("recovering",))+ ids = {}+ try:+ ids = {"recovering": _read(tmp_path / "recovering.json")}+ owners["recovering"].kill()+ owners["recovering"].wait(timeout=10)+ # The foreground command is not a job: it dies even here.+ wait_until(lambda: not any(_alive(i) for i in ids["recovering"]["foreground"]), timeout=15,+ what="the dead host's foreground command to exit")+ jobs = [i for name, job in ids["recovering"].items() if name != "foreground" for i in job]+ assert all(_alive(i) for i in jobs), (+ "a recovering host's background job died with it; the next host could not re-adopt it", jobs)finally:_cleanup(owners, log, ids)def _record_calls(monkeypatch):+ from tools import process_registry_ownership as registry_modfrom tools.environments import local as local_modcalls = []- monkeypatch.setattr(local_mod, "own_process_groups", lambda pgids: calls.append(("own", list(pgids))))- monkeypatch.setattr(local_mod, "release_process_groups", lambda pgids: calls.append(("release", list(pgids))))+ for mod in (registry_mod, local_mod):+ monkeypatch.setattr(mod, "own_process_groups", lambda pgids: calls.append(("own", list(pgids))))+ monkeypatch.setattr(mod, "release_process_groups", lambda pgids: calls.append(("release", list(pgids))))+ monkeypatch.setattr("hermes_cli.process_identity.own_process_groups",+ lambda pgids: calls.append(("own", list(pgids))))+ monkeypatch.setattr("hermes_cli.process_identity.register_child", lambda pid, purpose: None)return calls-def test_a_returned_command_releases_its_group(monkeypatch, tmp_path):+def test_a_finished_job_and_a_returned_command_release_their_group(monkeypatch, tmp_path):from tools.environments.local import LocalEnvironment+ from tools.process_registry import ProcessRegistrycalls = _record_calls(monkeypatch)+ registry = ProcessRegistry()+ session = registry.spawn_local("exit 0", cwd=str(tmp_path))+ wait_until(lambda: session.exited and ("release", [session.pid]) in calls, timeout=15,+ what="the finished job's release")+ assert calls[0] == ("own", [session.pid])++ calls.clear()env = LocalEnvironment(cwd=str(tmp_path), timeout=30)calls.clear()assert env.execute("echo hi")["returncode"] == 0assert [verb for verb, _ in calls] == ["own", "release"] and calls[0][1] == calls[1][1], calls+++def test_a_command_yielded_to_background_keeps_its_group_until_the_job_ends(monkeypatch, tmp_path):+ import threading++ from tools.environments.local import LocalEnvironment+ from tools.interrupt import request_yield+ from tools.process_registry import ProcessRegistry++ calls = _record_calls(monkeypatch)+ env = LocalEnvironment(cwd=str(tmp_path), timeout=60)+ registry = ProcessRegistry()+ gate, started = tmp_path / "gate", tmp_path / "started"+ adopted = []++ def handler(proc, output_so_far):+ adopted.append(registry.adopt_local(proc, command="wait for gate", cwd=str(tmp_path)))+ return {"output": output_so_far, "returncode": None}++ result = {}+ worker = threading.Thread(target=lambda: result.update(env.execute(+ f"touch {started}; while [ ! -e {gate} ]; do sleep 0.05; done", yield_handler=handler)))+ calls.clear()+ worker.start()+ wait_until(started.exists, timeout=15, what="the foreground command to start")+ request_yield(worker.ident)+ worker.join(timeout=15)+ assert result.get("returncode") is None and adopted, result+ session = adopted[0]+ assert calls == [("own", [session.pid])], "the yielding foreground wait released the job's group"+ assert session._owned_pgid == session.pid+ gate.touch()+ wait_until(lambda: ("release", [session.pid]) in calls, timeout=15, what="the adopted job's release")+ assert calls.count(("release", [session.pid])) == 1, calls+++def test_unowned_jobs_are_never_handed_to_the_supervisor(monkeypatch, tmp_path):+ from tools.process_registry import ProcessRegistry++ calls = _record_calls(monkeypatch)+ registry = ProcessRegistry()+ persisted = registry.spawn_local("exit 0", cwd=str(tmp_path), persist_on_release=True)+ registry._jobs_outlive_host = True+ recovered_host_job = registry.spawn_local("exit 0", cwd=str(tmp_path))+ wait_until(lambda: persisted.exited and recovered_host_job.exited, timeout=15, what="both jobs to exit")+ assert calls == [], callsdiff --git a/tools/environments/local.py b/tools/environments/local.pyindex 47b6ee2139c..4d0bd5ed215 100644--- a/tools/environments/local.py+++ b/tools/environments/local.py@@ -1171,7 +1171,8 @@ class LocalEnvironment(BaseEnvironment):def _wait_for_process(self, proc, *args, **kwargs) -> dict:"""Base wait, then hand the command's group back from the owner-death supervisor: once thecommand returned, what it left running in the background (`server &`) is not ours to kill,- exactly as on a graceful exit."""+ exactly as on a graceful exit. A command that yielded to background took the group with it+ (``ProcessRegistry.adopt_local`` clears ``_hermes_owned_pgid``)."""try:return super()._wait_for_process(proc, *args, **kwargs)finally:diff --git a/tools/process_registry.py b/tools/process_registry.pyindex 7f55acd6b87..52defc75c7a 100644--- a/tools/process_registry.py+++ b/tools/process_registry.py@@ -36,6 +36,7 @@ from tools.process_registry_notifications import (format_process_notification, heartbeat_output_tail, is_child_process_notification, should_surface_notification)from tools.process_registry_checkpoint import ProcessCheckpointMixinfrom tools.process_registry_output_log import ProcessOutputLogMixin, open_output_log, record_exit_command, worker_output_log+from tools.process_registry_ownership import ProcessOwnershipMixinfrom tools.process_registry_termination import ProcessTerminationMixinfrom tools.process_registry_results import load_completed_results, save_completed_resultfrom tools.process_registry_env_log import log_delta_command@@ -609,6 +610,7 @@ class ProcessSession:_reader_finish_requested: threading.Event = field(default_factory=threading.Event, repr=False)_reader_selectable: bool = field(default=False, repr=False)_pty: Any = field(default=None, repr=False) # ptyprocess handle (use_pty=True)+ _owned_pgid: Optional[int] = field(default=None, repr=False) # see ProcessOwnershipMixindef __post_init__(self):# A session built without an explicit owner is owned by its own task, so ownership checks compare@@ -672,7 +674,7 @@ _WSL_CHAIN_NOTE = ()-class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOutputLogMixin):+class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOutputLogMixin, ProcessOwnershipMixin):"""In-memory registry of running and finished background processes.Thread-safe: accessed from executor threads (terminal_tool, process handlers),the gateway asyncio loop (watchers, reset checks) and the cleanup thread."""@@ -1167,6 +1169,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOupty_env.setdefault("GIT_PAGER", "cat")pty_env.setdefault("PAGER", "cat")pty_proc = _PtyProcessCls.spawn(pty_argv, cwd=session.cwd, env=pty_env, dimensions=(30, 120))+ self._own_pty_job(session, pty_proc)session.pid = pty_proc.pidsession.host_start_time = self._safe_host_start_time(session.pid)session._pty = pty_proc@@ -1222,10 +1225,10 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu# the scope attaches to the invoked process, not the spawning session.output = open_output_log(log) if log else subprocess.PIPEtry:- proc = subprocess.Popen(- spawn_argv, text=True, cwd=session.cwd, env=spawn_env, encoding="utf-8",- errors="replace", stdout=output, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL,- start_new_session=True, **_popen_kwargs)+ proc = self._spawn_job(+ session, spawn_argv, persist_on_release, text=True, cwd=session.cwd, env=spawn_env,+ encoding="utf-8", errors="replace", stdout=output, stderr=subprocess.STDOUT,+ stdin=subprocess.DEVNULL, **_popen_kwargs)finally:if log:os.close(output)@@ -1239,21 +1242,6 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOuraisereturn session- def _reap_untracked(self, session: ProcessSession, proc: subprocess.Popen) -> None:- """Post-Popen setup failed: kill the orphaned subprocess (and any setsid- descendants) so nothing leaks untracked."""- with suppress(Exception):- if session.systemd_unit:- # Scope teardown is the authoritative cleanup for the worker cgroup;- # the systemd-run wrapper PID is terminated below as fallback.- _stop_systemd_unit(session.systemd_unit)- # _terminate_host_pid revalidates the kernel start time captured at- # spawn before signalling: pgid == pid would only prove the pid leads a- # group, and a reaped child can be recycled onto an unrelated leader.- self._terminate_host_pid(proc.pid, session.host_start_time)- with suppress(Exception):- proc.wait(timeout=5)-def adopt_local(self, proc: subprocess.Popen, *, command: str, cwd: Optional[str], task_id: str = "",session_key: str = "", owner_task_id: str = "", output_so_far: str = "",@@ -1268,6 +1256,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOusession.pid = proc.pidsession.host_start_time = self._safe_host_start_time(session.pid)session.notify_on_complete = notify_on_complete+ self._take_handoff_group(session, proc)if output_so_far:session.append_output(output_so_far)self._track_started(session, self._reader_loop, f"proc-reader-{session.id}")@@ -1567,6 +1556,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOusave_completed_result(session)self._running.pop(session.id)self._finished[session.id] = session+ self._release_job_group(session)# Release the retained Popen/PTY handles now: otherwise every# finished-but-unpruned session keeps its stdout pipe (or PTY master)# FD open until FINISHED_TTL_SECONDS elapses, and heavy backgrounddiff --git a/tools/process_registry_checkpoint.py b/tools/process_registry_checkpoint.pyindex c05ab07a5ea..176f0c3daba 100644--- a/tools/process_registry_checkpoint.py+++ b/tools/process_registry_checkpoint.py@@ -78,6 +78,9 @@ class ProcessCheckpointMixin:)from tools.process_registry_output_log import adoptable_log+ # This host adopts what a crashed predecessor left running, so the jobs it starts must+ # survive its own crash for the next one: they stay out of the owner-death supervisor.+ self._jobs_outlive_host = Truecheckpoint_path = _checkpoint_path()if not checkpoint_path.exists():return 0diff --git a/tools/process_registry_ownership.py b/tools/process_registry_ownership.pynew file mode 100644index 00000000000..22341ce825d--- /dev/null+++ b/tools/process_registry_ownership.py@@ -0,0 +1,82 @@+"""Which local background jobs die with their owner, for ProcessRegistry (process_identity layer 4)."""++import subprocess+import threading+from contextlib import suppress+from typing import Any++from hermes_cli.process_identity import own_process_groups, release_process_groups, spawn_owned+++class ProcessOwnershipMixin:+ """A local job's process group goes to the owner-death supervisor while the job's own process runs,+ so it dies with this process however this process dies (SIGKILL, OOM, crash): the agent-lifecycle+ sweeps only run while we are alive, and a killed CLI/TUI/worker left every job running on init.+ ``ProcessSession._owned_pgid`` is that group, cleared by the one release so a recycled pgid is+ never unregistered on someone else's behalf. The subclass supplies ``_terminate_host_pid``."""++ # Set by recover_from_checkpoint: this host re-adopts the jobs a crashed predecessor left+ # running, so its own jobs must outlive its crash the same way (see _owns_job).+ _jobs_outlive_host = False+ _owned_pgid_lock = threading.Lock()++ def _owns_job(self, persist_on_release: bool) -> bool:+ """Not owned, on purpose: ``persist_on_release`` jobs, which must outlive the agent (#41225);+ every job of a gateway, which re-adopts its jobs from the checkpoint after a crash (flagged by+ recover_from_checkpoint; a supervised gateway also before it gets there); and Windows, where+ the layer-3 job object covers ownership."""+ from tools.process_registry import _IS_WINDOWS, _is_supervised_gateway_process++ return not (_IS_WINDOWS or persist_on_release or self._jobs_outlive_host+ or _is_supervised_gateway_process())++ def _spawn_job(self, session: Any, argv: list[str], persist_on_release: bool, **popen_kwargs) -> subprocess.Popen:+ """Start a pipe job in its own session, owned unless ``_owns_job`` says it must outlive us."""+ if self._owns_job(persist_on_release):+ proc = spawn_owned(argv, purpose="terminal-background", **popen_kwargs)+ session._owned_pgid = proc.pid+ return proc+ # noqa: subprocess-stdin — a forwarder: spawn_local passes stdin=DEVNULL in popen_kwargs+ return subprocess.Popen(argv, start_new_session=True, **popen_kwargs) # detached-spawn: ok — see _owns_job++ def _own_pty_job(self, session: Any, pty_proc: Any) -> None:+ """The PTY child leads its own session and group (pty.fork). Its hangup reaches only jobs that+ keep the default SIGHUP action; `nohup`/trap-HUP servers outlived a killed owner."""+ if self._owns_job(session.persist_on_release):+ own_process_groups([pty_proc.pid])+ session._owned_pgid = pty_proc.pid++ def _take_handoff_group(self, session: Any, proc: subprocess.Popen) -> None:+ """A foreground command that yields to background hands its supervised group over with the+ pipe (LocalEnvironment._run_bash): the job keeps it, unless jobs here must outlive us."""+ owned_pgid, proc._hermes_owned_pgid = getattr(proc, "_hermes_owned_pgid", None), None+ if owned_pgid is not None and self._owns_job(False):+ session._owned_pgid = owned_pgid+ elif owned_pgid is not None:+ release_process_groups([owned_pgid])++ def _release_job_group(self, session: Any) -> None:+ """The job's own process exited or was killed: stop covering its group. What it left behind on+ purpose (`server &`) is no longer this job's, exactly as before."""+ with self._owned_pgid_lock:+ owned_pgid, session._owned_pgid = session._owned_pgid, None+ if owned_pgid is not None:+ release_process_groups([owned_pgid])++ def _reap_untracked(self, session: Any, proc: subprocess.Popen) -> None:+ """Post-Popen setup failed: kill the orphaned subprocess (and any setsid+ descendants) so nothing leaks untracked."""+ from tools.process_registry import _stop_systemd_unit++ with suppress(Exception):+ if session.systemd_unit:+ # Scope teardown is the authoritative cleanup for the worker cgroup;+ # the systemd-run wrapper PID is terminated below as fallback.+ _stop_systemd_unit(session.systemd_unit)+ # _terminate_host_pid revalidates the kernel start time captured at+ # spawn before signalling: pgid == pid would only prove the pid leads a+ # group, and a reaped child can be recycled onto an unrelated leader.+ self._terminate_host_pid(proc.pid, session.host_start_time)+ with suppress(Exception):+ proc.wait(timeout=5)+ self._release_job_group(session)