The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(scratch): the detached prune spares the launcher's whole ancestry and fails closed
Teknium6cf8173a2b07
diff --git a/hermes_constants_scratch.py b/hermes_constants_scratch.pyindex 57d3ec7495b..bce8b7cd4c8 100644--- a/hermes_constants_scratch.py+++ b/hermes_constants_scratch.py@@ -160,36 +160,56 @@ def _under(path: str, root: str) -> bool:return path == root or path.startswith(root.rstrip(os.sep) + os.sep)-# A detached pass (hermes_constants_scratch_detach) is reparented once the Hermes command that-# launched it exits; that command's ancestors, captured when the pass starts, stay its own.-_launcher_lineage: set[int] = set()+# The ancestry of the Hermes command that launched a detached pass (hermes_constants_scratch_detach),+# as ``(pid, create_time)`` pairs the launcher computed before it spawned the pass: the pass is+# reparented once that command exits, so its own ancestry no longer reaches the user's shell.+# ``None``: the launcher could not compute it, so the pass signals nothing and keeps every entry+# a live process works in.+_launcher_lineage: frozenset[tuple[int, float]] | None = frozenset()+++def lineage_with_start_times() -> list[tuple[int, float]] | None:+ """This process and its ancestors with their create times, the set :func:`_own_lineage`+ computes in-process, in a form that stays checkable after they exit. ``None`` when it+ cannot be computed completely (no psutil, a process that refuses inspection)."""+ try:+ import psutil+ except ImportError:+ return None+ chain: dict[int, float] = {}+ try:+ proc = psutil.Process()+ while proc is not None and proc.pid not in chain:+ chain[proc.pid] = proc.create_time()+ proc = proc.parent()+ except (psutil.Error, OSError):+ return None+ return list(chain.items())-def _lineage(pid: int | None = None) -> set[int]:+def _own_lineage() -> set[int] | None:+ """This process and its ancestors: never reap the shell that is running the prune. A+ detached pass adds its launcher's ancestors that still run under the same create time (a+ reused pid is not spared), and returns ``None`` (spare every process) without them."""import psutil+ if _launcher_lineage is None:+ return Nonepids: set[int] = set()try:- proc = psutil.Process(pid)+ proc = psutil.Process()while proc is not None and proc.pid not in pids:pids.add(proc.pid)proc = proc.parent()except (psutil.Error, OSError):pass+ for pid, started in _launcher_lineage:+ with contextlib.suppress(psutil.Error, OSError):+ if psutil.Process(pid).create_time() == started:+ pids.add(pid)return pids-def adopt_launcher_lineage(pids: tuple[int, ...]) -> None:- for pid in pids:- _launcher_lineage.update(_lineage(pid))---def _own_lineage() -> set[int]:- """This process and its ancestors (and its launcher's): never reap the shell that is- running the prune."""- return _lineage() | _launcher_lineage--def _registered_live_pids(home: Path) -> set[int]:"""Live pids Hermes registered for *home*: the machine spawn ledger (every long-lived entrypoint and MCP helper, ``hermes_cli.process_identity``) and the background-process checkpoint@@ -282,7 +302,7 @@ def reap_processes_rooted_in(scratch_root: Path, doomed: list[Path], audit: loggif not _under(cwd_path, root):continueif deleted or not os.path.exists(cwd_path) or any(_under(cwd_path, t) for t in targets):- (lineage if proc.pid in skip else victims).append((proc, cwd_path))+ (lineage if skip is None or proc.pid in skip else victims).append((proc, cwd_path))# The owning home comes from the scratch path as configured, not its resolved target:# a cache/scratch symlinked onto another disk still belongs to <home>.registered = _registered_live_pids(Path(scratch_root).absolute().parent.parent) if victims else set()@@ -406,8 +426,11 @@ def _prune_idle_entries(# (or by hand) are found by the deleted-cwd rule, not by membership in ``doomed``.try:reap_processes_rooted_in(root, doomed, audit)- except Exception as exc: # psutil missing or restricted host: the deletion still proceeds- logger.debug("scratch prune: process reap skipped: %s", exc, exc_info=True)+ # health: allow BLE001 -- fail-closed boundary: any failure leaves who works inside unknown; audited below+ except Exception as exc: # psutil missing or restricted host+ audit.info("scratch prune: process check failed (%s: %r), nothing deleted this pass",+ type(exc).__name__, str(exc))+ return 0if not doomed:return 0repos: set[str] = set()diff --git a/hermes_constants_scratch_detach.py b/hermes_constants_scratch_detach.pyindex f21411c51e3..11f945022da 100644--- a/hermes_constants_scratch_detach.py+++ b/hermes_constants_scratch_detach.py@@ -13,25 +13,30 @@ import subprocessimport sysfrom pathlib import Path+import hermes_constants_scratch as rules+logger = logging.getLogger(__name__)# ``-I``: neither the launcher's cwd (a project ``utils/`` would shadow ours) nor PYTHON* env vars# decide what the child imports; it gets the launcher's own ``sys.path`` (repo root, PM/venv# site-packages), not ``hermes_bootstrap``, whose boot hook would schedule another prune.-_CHILD = ("import sys; sys.path[:] = sys.argv[4:]; "- "import hermes_constants_scratch_detach as d; d.run(*sys.argv[1:4])")+_CHILD = ("import sys; sys.path[:] = sys.argv[3:]; "+ "import hermes_constants_scratch_detach as d; d.run(*sys.argv[1:3])")def spawn_prune(scratch: Path) -> None:"""Start one detached prune of *scratch*, the path the caller resolved (a child neverre-resolves the profile home). A failed spawn is logged and dropped: the stamp is already- spent, so the next pass is the next process past the hourly mark."""+ spent, so the next pass is the next process past the hourly mark.++ This process's ancestry is captured here, before the spawn: once this command and its+ parents exit, the child can no longer see that the user's shell was the caller's."""argv = [sys.executable, "-I"]if sys.dont_write_bytecode:argv.append("-B")if sys.pycache_prefix: # a sealed payload must never get __pycache__ writesargv += ["-X", f"pycache_prefix={sys.pycache_prefix}"]- argv += ["-c", _CHILD, os.fspath(scratch), str(os.getpid()), str(os.getppid()), *sys.path]+ argv += ["-c", _CHILD, os.fspath(scratch), _encode_lineage(rules.lineage_with_start_times()), *sys.path]from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway, windows_detach_popen_kwargsstdio = {"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}@@ -47,11 +52,24 @@ def spawn_prune(scratch: Path) -> None:logger.debug("scratch prune: could not start the detached pass for %s: %s", scratch, exc)-def run(scratch: str, launcher_pid: str, launcher_parent_pid: str) -> None:- """Child entry. The launcher's process tree stays the pass's own (never reaped; an entry it- works inside is kept) after the launcher exits and this child is reparented away from it."""- import hermes_constants_scratch- from hermes_constants import prune_scratch_dir+def _encode_lineage(chain: list[tuple[int, float]] | None) -> str:+ # repr() round-trips a float exactly; "-" is "could not compute" (the child fails closed).+ return "-" if chain is None else ",".join(f"{pid}:{started!r}" for pid, started in chain)+++def run(scratch: str, lineage: str) -> None:+ """Child entry: one prune of *scratch*, sparing the launcher's ancestry. Any failure is one+ line in ``scratch-prune.log``; nothing is raised (no one reads this process's stderr)."""+ try:+ from hermes_constants import prune_scratch_dir- hermes_constants_scratch.adopt_launcher_lineage((int(launcher_pid), int(launcher_parent_pid)))- prune_scratch_dir(Path(scratch))+ rules._launcher_lineage = None if lineage == "-" else frozenset(+ (int(pid), float(started)) for pid, started in (item.split(":") for item in lineage.split(",")))+ prune_scratch_dir(Path(scratch))+ # health: allow BLE001 -- the detached child's top level: its stderr is /dev/null, so the audit log is the record+ except Exception as exc:+ audit = rules._open_prune_log(Path(scratch).parent.parent / "logs" / "scratch-prune.log")+ audit.info("scratch prune: detached pass failed (%s: %r)", type(exc).__name__, str(exc))+ if audit is not rules.logger:+ for handler in audit.handlers:+ handler.close()diff --git a/tests/conftest.py b/tests/conftest.pyindex cda6c3367fe..8b74a46aaa7 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -121,9 +121,9 @@ def _hermes_home_points_at_production(value: str) -> bool:from hermes_constants import SCRATCH_DIR_MARKER_ENV, SCRATCH_TMP_ENV_VARSimport hermes_constants-# The hourly scratch prune runs in a detached child process; a test reaching get_scratch_dir()-# would start a real one (and hand it to any Popen mock it installed). Tests of the prune-# reset this flag.+# The hourly scratch prune runs in a detached child process. An import-time boot hook during+# collection must not start one: mark this process's once-per-process prune done. Tests that+# reload hermes_constants reset that flag, so the per-test fixture below stubs the spawn itself.hermes_constants._scratch_pruned_once = True_HERMES_EXPORTED_TMP = os.environ.get(SCRATCH_DIR_MARKER_ENV, "")@@ -709,6 +709,18 @@ def _kanban_write_guard(_hermetic_environment, monkeypatch):monkeypatch.setattr(_kdbc, "connect", _guarded_connect)[email protected](autouse=True)+def scratch_prune_spawns(monkeypatch):+ """No test starts a real detached scratch prune (one would outlive the test, and get any+ Popen mock the test installed): ``spawn_prune`` only records the scratch dir it was given.+ A test that needs the real child runs a launcher in a subprocess."""+ import hermes_constants_scratch_detach++ spawned: list = []+ monkeypatch.setattr(hermes_constants_scratch_detach, "spawn_prune", spawned.append)+ return spawned++# ── Live state.db write guard ───────────────────────────────────────────────# Companion to the kanban guard above, for the MAIN state database.# ``hermes_state._ensure_test_isolation`` (the single choke point everydiff --git a/tests/test_scratch_dir.py b/tests/test_scratch_dir.pyindex dc7db6db945..546a1286739 100644--- a/tests/test_scratch_dir.py+++ b/tests/test_scratch_dir.py@@ -19,26 +19,14 @@ import pytestfrom hermes_constants import apply_scratch_tmp_env, get_scratch_dir, prune_scratch_dir-def spawned_prunes(monkeypatch):- """A once-per-process prune that is still due; records each process spawned for it."""- import hermes_constants as constants-- monkeypatch.setattr(constants, "_scratch_pruned_once", False)- spawned = []- monkeypatch.setattr(subprocess, "Popen", lambda argv, **kwargs: spawned.append(argv))- token = constants.set_hermes_home_override(None)- yield spawned- constants.reset_hermes_home_override(token)---def test_boot_hook_and_prompt_hand_the_prune_to_one_child(spawned_prunes, tmp_path, monkeypatch):+def test_boot_hook_and_prompt_hand_the_prune_to_one_child(scratch_prune_spawns, tmp_path, monkeypatch):"""The boot hook returns with TMPDIR pointing at a scratch dir that already exists andstarts exactly one prune process, for that dir; the prompt build starts no other. A routedcaller's process prunes the routed home's dir, never the launch home's."""import hermes_constants as constantsfrom agent.prompt_builder import _local_host_hints+ monkeypatch.setattr(constants, "_scratch_pruned_once", False)home = tmp_path / "launch"monkeypatch.setenv("HERMES_HOME", str(home))scratch = home / "cache" / "scratch"@@ -47,7 +35,7 @@ def test_boot_hook_and_prompt_hand_the_prune_to_one_child(spawned_prunes, tmp_pahints = _local_host_hints()assert env["TMPDIR"] == str(scratch) and scratch.is_dir()assert f"Scratch directory: {scratch} (TMPDIR points here" in hints[0]- assert len(spawned_prunes) == 1 and str(scratch) in spawned_prunes[0]+ assert scratch_prune_spawns == [scratch]monkeypatch.setattr(constants, "_scratch_pruned_once", False) # the routed caller's own processrouted = tmp_path / "routed"@@ -56,8 +44,7 @@ def test_boot_hook_and_prompt_hand_the_prune_to_one_child(spawned_prunes, tmp_paassert get_scratch_dir() == routed / "cache" / "scratch"finally:constants.reset_hermes_home_override(token)- assert len(spawned_prunes) == 2 and str(routed / "cache" / "scratch") in spawned_prunes[1]- assert str(scratch) not in spawned_prunes[1]+ assert scratch_prune_spawns == [scratch, routed / "cache" / "scratch"]def _idle_entry(scratch: Path, name: str) -> Path:@@ -73,20 +60,32 @@ def _idle_entry(scratch: Path, name: str) -> Path:def test_a_short_lived_process_still_completes_its_due_prune(tmp_path):"""A `hermes` command that exits right after boot still gets its pass: the detached childremoves the idle entry of the (routed) home the command resolved, after the command is gone,- and leaves the launch home alone. Red on a daemon-thread prune, which dies with the command."""+ and leaves the launch home alone. Red on a daemon-thread prune, which dies with the command.+ The user's shell stays the pass's own even when everything between it and the pass exited+ first: a shell working inside an idle entry is neither signalled nor has its entry removed."""launch, routed = tmp_path / "launch", tmp_path / "routed"kept = _idle_entry(get_scratch_dir(launch, prune=False), "launch-lane")doomed = _idle_entry(get_scratch_dir(routed, prune=False), "routed-lane")+ shell_entry = _idle_entry(get_scratch_dir(routed, prune=False), "shell-lane")repo = Path(__file__).resolve().parents[1]launcher = ("import sys; sys.path.insert(0, sys.argv[1]); from pathlib import Path; import hermes_constants as h; ""h.set_hermes_home_override(Path(sys.argv[2])); h.get_scratch_dir()")+ # shell (cwd in an idle entry, stays) -> wrapper (exits once the launcher does) -> launcher+ run = "import subprocess, sys; subprocess.run(sys.argv[1:], cwd={!r}, stdin=subprocess.DEVNULL, check=True)"+ wrapper = [sys.executable, "-c", run.format(str(tmp_path)), sys.executable, "-c", launcher, str(repo), str(routed)]+ shell = [sys.executable, "-c", run.format(str(tmp_path)) + "; import time; time.sleep(120)", *wrapper]env = {**os.environ, "HERMES_HOME": str(launch)}- subprocess.run([sys.executable, "-c", launcher, str(repo), str(routed)], env=env, cwd=tmp_path,- stdin=subprocess.DEVNULL, check=True, timeout=60)- wait_until(lambda: not doomed.exists(), timeout=60, what="the detached prune removing the idle entry")- assert kept.exists()- log = (routed / "logs" / "scratch-prune.log").read_text(encoding="utf-8-sig")- assert f"removed {str(doomed)!r} (" in log, log+ user_shell = subprocess.Popen(shell, env=env, cwd=shell_entry / "sub", stdin=subprocess.DEVNULL)+ try:+ wait_until(lambda: not doomed.exists(), timeout=60, what="the detached prune removing the idle entry")+ assert kept.exists() and shell_entry.exists()+ assert user_shell.poll() is None, "the user's shell was signalled"+ log = (routed / "logs" / "scratch-prune.log").read_text(encoding="utf-8-sig")+ assert f"removed {str(doomed)!r} (" in log, log+ assert f"kept {str(shell_entry)!r} — live Hermes process pid={user_shell.pid}" in log, log+ finally:+ user_shell.kill()+ user_shell.wait(timeout=30)def test_scratch_env_follows_home_and_respects_user_tmpdir(tmp_path):diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.mdindex 137e0c77499..0b841f59c20 100644--- a/website/docs/user-guide/configuration.md+++ b/website/docs/user-guide/configuration.md@@ -268,8 +268,10 @@ with its working directory inside that entry (or inside a scratch path that no lsuch as a headless browser left behind by a test run) and drops any `git worktree`registration that pointed into it. A process Hermes itself started and still tracks (abackground `terminal` process, the gateway, a CLI session, or anything they launched that is-still their child) is never stopped: an idle entry one of them is working inside is kept-instead. Each entry is re-checked right before it is deleted, so+still their child) is never stopped, and neither is the shell that started the Hermes command+(or its parents), even after that command exited: an idle entry one of them is working inside+is kept instead. A pass that cannot inspect processes stops nothing and deletes no entry a+process may be working in. Each entry is re-checked right before it is deleted, soone written to while the prune was stopping processes is kept. Every removal, stoppedprocess and kept entry is recorded in `~/.hermes/logs/scratch-prune.log`. An entry holdingfiles your user may not read or delete (typically root-owned output of a Docker container that