Skip to content

The agent that grows with you

fix(scratch): the detached prune spares the launcher's whole ancestry and fails closed

Teknium6cf8173a2b07
diff --git a/hermes_constants_scratch.py b/hermes_constants_scratch.py
index 57d3ec7495b..bce8b7cd4c8 100644
--- a/hermes_constants_scratch.py
+++ b/hermes_constants_scratch.py
@@ -160,36 +160,56 @@ def _under(path: str, root: str) -> bool:
return path == root or path.startswith(root.rstrip(os.sep) + os.sep)
-# A detached pass (hermes_constants_scratch_detach) is reparented once the Hermes command that
-# launched it exits; that command's ancestors, captured when the pass starts, stay its own.
-_launcher_lineage: set[int] = set()
+# The ancestry of the Hermes command that launched a detached pass (hermes_constants_scratch_detach),
+# as ``(pid, create_time)`` pairs the launcher computed before it spawned the pass: the pass is
+# reparented once that command exits, so its own ancestry no longer reaches the user's shell.
+# ``None``: the launcher could not compute it, so the pass signals nothing and keeps every entry
+# a live process works in.
+_launcher_lineage: frozenset[tuple[int, float]] | None = frozenset()
+
+
+def lineage_with_start_times() -> list[tuple[int, float]] | None:
+ """This process and its ancestors with their create times, the set :func:`_own_lineage`
+ computes in-process, in a form that stays checkable after they exit. ``None`` when it
+ cannot be computed completely (no psutil, a process that refuses inspection)."""
+ try:
+ import psutil
+ except ImportError:
+ return None
+ chain: dict[int, float] = {}
+ try:
+ proc = psutil.Process()
+ while proc is not None and proc.pid not in chain:
+ chain[proc.pid] = proc.create_time()
+ proc = proc.parent()
+ except (psutil.Error, OSError):
+ return None
+ return list(chain.items())
-def _lineage(pid: int | None = None) -> set[int]:
+def _own_lineage() -> set[int] | None:
+ """This process and its ancestors: never reap the shell that is running the prune. A
+ detached pass adds its launcher's ancestors that still run under the same create time (a
+ reused pid is not spared), and returns ``None`` (spare every process) without them."""
import psutil
+ if _launcher_lineage is None:
+ return None
pids: set[int] = set()
try:
- proc = psutil.Process(pid)
+ proc = psutil.Process()
while proc is not None and proc.pid not in pids:
pids.add(proc.pid)
proc = proc.parent()
except (psutil.Error, OSError):
pass
+ for pid, started in _launcher_lineage:
+ with contextlib.suppress(psutil.Error, OSError):
+ if psutil.Process(pid).create_time() == started:
+ pids.add(pid)
return pids
-def adopt_launcher_lineage(pids: tuple[int, ...]) -> None:
- for pid in pids:
- _launcher_lineage.update(_lineage(pid))
-
-
-def _own_lineage() -> set[int]:
- """This process and its ancestors (and its launcher's): never reap the shell that is
- running the prune."""
- return _lineage() | _launcher_lineage
-
-
def _registered_live_pids(home: Path) -> set[int]:
"""Live pids Hermes registered for *home*: the machine spawn ledger (every long-lived entry
point and MCP helper, ``hermes_cli.process_identity``) and the background-process checkpoint
@@ -282,7 +302,7 @@ def reap_processes_rooted_in(scratch_root: Path, doomed: list[Path], audit: logg
if not _under(cwd_path, root):
continue
if deleted or not os.path.exists(cwd_path) or any(_under(cwd_path, t) for t in targets):
- (lineage if proc.pid in skip else victims).append((proc, cwd_path))
+ (lineage if skip is None or proc.pid in skip else victims).append((proc, cwd_path))
# The owning home comes from the scratch path as configured, not its resolved target:
# a cache/scratch symlinked onto another disk still belongs to <home>.
registered = _registered_live_pids(Path(scratch_root).absolute().parent.parent) if victims else set()
@@ -406,8 +426,11 @@ def _prune_idle_entries(
# (or by hand) are found by the deleted-cwd rule, not by membership in ``doomed``.
try:
reap_processes_rooted_in(root, doomed, audit)
- except Exception as exc: # psutil missing or restricted host: the deletion still proceeds
- logger.debug("scratch prune: process reap skipped: %s", exc, exc_info=True)
+ # health: allow BLE001 -- fail-closed boundary: any failure leaves who works inside unknown; audited below
+ except Exception as exc: # psutil missing or restricted host
+ audit.info("scratch prune: process check failed (%s: %r), nothing deleted this pass",
+ type(exc).__name__, str(exc))
+ return 0
if not doomed:
return 0
repos: set[str] = set()
diff --git a/hermes_constants_scratch_detach.py b/hermes_constants_scratch_detach.py
index f21411c51e3..11f945022da 100644
--- a/hermes_constants_scratch_detach.py
+++ b/hermes_constants_scratch_detach.py
@@ -13,25 +13,30 @@ import subprocess
import sys
from pathlib import Path
+import hermes_constants_scratch as rules
+
logger = logging.getLogger(__name__)
# ``-I``: neither the launcher's cwd (a project ``utils/`` would shadow ours) nor PYTHON* env vars
# decide what the child imports; it gets the launcher's own ``sys.path`` (repo root, PM/venv
# site-packages), not ``hermes_bootstrap``, whose boot hook would schedule another prune.
-_CHILD = ("import sys; sys.path[:] = sys.argv[4:]; "
- "import hermes_constants_scratch_detach as d; d.run(*sys.argv[1:4])")
+_CHILD = ("import sys; sys.path[:] = sys.argv[3:]; "
+ "import hermes_constants_scratch_detach as d; d.run(*sys.argv[1:3])")
def spawn_prune(scratch: Path) -> None:
"""Start one detached prune of *scratch*, the path the caller resolved (a child never
re-resolves the profile home). A failed spawn is logged and dropped: the stamp is already
- spent, so the next pass is the next process past the hourly mark."""
+ spent, so the next pass is the next process past the hourly mark.
+
+ This process's ancestry is captured here, before the spawn: once this command and its
+ parents exit, the child can no longer see that the user's shell was the caller's."""
argv = [sys.executable, "-I"]
if sys.dont_write_bytecode:
argv.append("-B")
if sys.pycache_prefix: # a sealed payload must never get __pycache__ writes
argv += ["-X", f"pycache_prefix={sys.pycache_prefix}"]
- argv += ["-c", _CHILD, os.fspath(scratch), str(os.getpid()), str(os.getppid()), *sys.path]
+ argv += ["-c", _CHILD, os.fspath(scratch), _encode_lineage(rules.lineage_with_start_times()), *sys.path]
from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway, windows_detach_popen_kwargs
stdio = {"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
@@ -47,11 +52,24 @@ def spawn_prune(scratch: Path) -> None:
logger.debug("scratch prune: could not start the detached pass for %s: %s", scratch, exc)
-def run(scratch: str, launcher_pid: str, launcher_parent_pid: str) -> None:
- """Child entry. The launcher's process tree stays the pass's own (never reaped; an entry it
- works inside is kept) after the launcher exits and this child is reparented away from it."""
- import hermes_constants_scratch
- from hermes_constants import prune_scratch_dir
+def _encode_lineage(chain: list[tuple[int, float]] | None) -> str:
+ # repr() round-trips a float exactly; "-" is "could not compute" (the child fails closed).
+ return "-" if chain is None else ",".join(f"{pid}:{started!r}" for pid, started in chain)
+
+
+def run(scratch: str, lineage: str) -> None:
+ """Child entry: one prune of *scratch*, sparing the launcher's ancestry. Any failure is one
+ line in ``scratch-prune.log``; nothing is raised (no one reads this process's stderr)."""
+ try:
+ from hermes_constants import prune_scratch_dir
- hermes_constants_scratch.adopt_launcher_lineage((int(launcher_pid), int(launcher_parent_pid)))
- prune_scratch_dir(Path(scratch))
+ rules._launcher_lineage = None if lineage == "-" else frozenset(
+ (int(pid), float(started)) for pid, started in (item.split(":") for item in lineage.split(",")))
+ prune_scratch_dir(Path(scratch))
+ # health: allow BLE001 -- the detached child's top level: its stderr is /dev/null, so the audit log is the record
+ except Exception as exc:
+ audit = rules._open_prune_log(Path(scratch).parent.parent / "logs" / "scratch-prune.log")
+ audit.info("scratch prune: detached pass failed (%s: %r)", type(exc).__name__, str(exc))
+ if audit is not rules.logger:
+ for handler in audit.handlers:
+ handler.close()
diff --git a/tests/conftest.py b/tests/conftest.py
index cda6c3367fe..8b74a46aaa7 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -121,9 +121,9 @@ def _hermes_home_points_at_production(value: str) -> bool:
from hermes_constants import SCRATCH_DIR_MARKER_ENV, SCRATCH_TMP_ENV_VARS
import hermes_constants
-# The hourly scratch prune runs in a detached child process; a test reaching get_scratch_dir()
-# would start a real one (and hand it to any Popen mock it installed). Tests of the prune
-# reset this flag.
+# The hourly scratch prune runs in a detached child process. An import-time boot hook during
+# collection must not start one: mark this process's once-per-process prune done. Tests that
+# reload hermes_constants reset that flag, so the per-test fixture below stubs the spawn itself.
hermes_constants._scratch_pruned_once = True
_HERMES_EXPORTED_TMP = os.environ.get(SCRATCH_DIR_MARKER_ENV, "")
@@ -709,6 +709,18 @@ def _kanban_write_guard(_hermetic_environment, monkeypatch):
monkeypatch.setattr(_kdbc, "connect", _guarded_connect)
[email protected](autouse=True)
+def scratch_prune_spawns(monkeypatch):
+ """No test starts a real detached scratch prune (one would outlive the test, and get any
+ Popen mock the test installed): ``spawn_prune`` only records the scratch dir it was given.
+ A test that needs the real child runs a launcher in a subprocess."""
+ import hermes_constants_scratch_detach
+
+ spawned: list = []
+ monkeypatch.setattr(hermes_constants_scratch_detach, "spawn_prune", spawned.append)
+ return spawned
+
+
# ── Live state.db write guard ───────────────────────────────────────────────
# Companion to the kanban guard above, for the MAIN state database.
# ``hermes_state._ensure_test_isolation`` (the single choke point every
diff --git a/tests/test_scratch_dir.py b/tests/test_scratch_dir.py
index dc7db6db945..546a1286739 100644
--- a/tests/test_scratch_dir.py
+++ b/tests/test_scratch_dir.py
@@ -19,26 +19,14 @@ import pytest
from hermes_constants import apply_scratch_tmp_env, get_scratch_dir, prune_scratch_dir
-def spawned_prunes(monkeypatch):
- """A once-per-process prune that is still due; records each process spawned for it."""
- import hermes_constants as constants
-
- monkeypatch.setattr(constants, "_scratch_pruned_once", False)
- spawned = []
- monkeypatch.setattr(subprocess, "Popen", lambda argv, **kwargs: spawned.append(argv))
- token = constants.set_hermes_home_override(None)
- yield spawned
- constants.reset_hermes_home_override(token)
-
-
-def test_boot_hook_and_prompt_hand_the_prune_to_one_child(spawned_prunes, tmp_path, monkeypatch):
+def test_boot_hook_and_prompt_hand_the_prune_to_one_child(scratch_prune_spawns, tmp_path, monkeypatch):
"""The boot hook returns with TMPDIR pointing at a scratch dir that already exists and
starts exactly one prune process, for that dir; the prompt build starts no other. A routed
caller's process prunes the routed home's dir, never the launch home's."""
import hermes_constants as constants
from agent.prompt_builder import _local_host_hints
+ monkeypatch.setattr(constants, "_scratch_pruned_once", False)
home = tmp_path / "launch"
monkeypatch.setenv("HERMES_HOME", str(home))
scratch = home / "cache" / "scratch"
@@ -47,7 +35,7 @@ def test_boot_hook_and_prompt_hand_the_prune_to_one_child(spawned_prunes, tmp_pa
hints = _local_host_hints()
assert env["TMPDIR"] == str(scratch) and scratch.is_dir()
assert f"Scratch directory: {scratch} (TMPDIR points here" in hints[0]
- assert len(spawned_prunes) == 1 and str(scratch) in spawned_prunes[0]
+ assert scratch_prune_spawns == [scratch]
monkeypatch.setattr(constants, "_scratch_pruned_once", False) # the routed caller's own process
routed = tmp_path / "routed"
@@ -56,8 +44,7 @@ def test_boot_hook_and_prompt_hand_the_prune_to_one_child(spawned_prunes, tmp_pa
assert get_scratch_dir() == routed / "cache" / "scratch"
finally:
constants.reset_hermes_home_override(token)
- assert len(spawned_prunes) == 2 and str(routed / "cache" / "scratch") in spawned_prunes[1]
- assert str(scratch) not in spawned_prunes[1]
+ assert scratch_prune_spawns == [scratch, routed / "cache" / "scratch"]
def _idle_entry(scratch: Path, name: str) -> Path:
@@ -73,20 +60,32 @@ def _idle_entry(scratch: Path, name: str) -> Path:
def test_a_short_lived_process_still_completes_its_due_prune(tmp_path):
"""A `hermes` command that exits right after boot still gets its pass: the detached child
removes the idle entry of the (routed) home the command resolved, after the command is gone,
- and leaves the launch home alone. Red on a daemon-thread prune, which dies with the command."""
+ and leaves the launch home alone. Red on a daemon-thread prune, which dies with the command.
+ The user's shell stays the pass's own even when everything between it and the pass exited
+ first: a shell working inside an idle entry is neither signalled nor has its entry removed."""
launch, routed = tmp_path / "launch", tmp_path / "routed"
kept = _idle_entry(get_scratch_dir(launch, prune=False), "launch-lane")
doomed = _idle_entry(get_scratch_dir(routed, prune=False), "routed-lane")
+ shell_entry = _idle_entry(get_scratch_dir(routed, prune=False), "shell-lane")
repo = Path(__file__).resolve().parents[1]
launcher = ("import sys; sys.path.insert(0, sys.argv[1]); from pathlib import Path; import hermes_constants as h; "
"h.set_hermes_home_override(Path(sys.argv[2])); h.get_scratch_dir()")
+ # shell (cwd in an idle entry, stays) -> wrapper (exits once the launcher does) -> launcher
+ run = "import subprocess, sys; subprocess.run(sys.argv[1:], cwd={!r}, stdin=subprocess.DEVNULL, check=True)"
+ wrapper = [sys.executable, "-c", run.format(str(tmp_path)), sys.executable, "-c", launcher, str(repo), str(routed)]
+ shell = [sys.executable, "-c", run.format(str(tmp_path)) + "; import time; time.sleep(120)", *wrapper]
env = {**os.environ, "HERMES_HOME": str(launch)}
- subprocess.run([sys.executable, "-c", launcher, str(repo), str(routed)], env=env, cwd=tmp_path,
- stdin=subprocess.DEVNULL, check=True, timeout=60)
- wait_until(lambda: not doomed.exists(), timeout=60, what="the detached prune removing the idle entry")
- assert kept.exists()
- log = (routed / "logs" / "scratch-prune.log").read_text(encoding="utf-8-sig")
- assert f"removed {str(doomed)!r} (" in log, log
+ user_shell = subprocess.Popen(shell, env=env, cwd=shell_entry / "sub", stdin=subprocess.DEVNULL)
+ try:
+ wait_until(lambda: not doomed.exists(), timeout=60, what="the detached prune removing the idle entry")
+ assert kept.exists() and shell_entry.exists()
+ assert user_shell.poll() is None, "the user's shell was signalled"
+ log = (routed / "logs" / "scratch-prune.log").read_text(encoding="utf-8-sig")
+ assert f"removed {str(doomed)!r} (" in log, log
+ assert f"kept {str(shell_entry)!r} — live Hermes process pid={user_shell.pid}" in log, log
+ finally:
+ user_shell.kill()
+ user_shell.wait(timeout=30)
def test_scratch_env_follows_home_and_respects_user_tmpdir(tmp_path):
diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md
index 137e0c77499..0b841f59c20 100644
--- a/website/docs/user-guide/configuration.md
+++ b/website/docs/user-guide/configuration.md
@@ -268,8 +268,10 @@ with its working directory inside that entry (or inside a scratch path that no l
such as a headless browser left behind by a test run) and drops any `git worktree`
registration that pointed into it. A process Hermes itself started and still tracks (a
background `terminal` process, the gateway, a CLI session, or anything they launched that is
-still their child) is never stopped: an idle entry one of them is working inside is kept
-instead. Each entry is re-checked right before it is deleted, so
+still their child) is never stopped, and neither is the shell that started the Hermes command
+(or its parents), even after that command exited: an idle entry one of them is working inside
+is kept instead. A pass that cannot inspect processes stops nothing and deletes no entry a
+process may be working in. Each entry is re-checked right before it is deleted, so
one written to while the prune was stopping processes is kept. Every removal, stopped
process and kept entry is recorded in `~/.hermes/logs/scratch-prune.log`. An entry holding
files your user may not read or delete (typically root-owned output of a Docker container that