Skip to content

The agent that grows with you

fix(scratch): stop re-walking and re-logging entries this user cannot delete

Teknium78e298e94f83
diff --git a/hermes_cli/doctor_state.py b/hermes_cli/doctor_state.py
index 7e34f19c3b4..55a21d34061 100644
--- a/hermes_cli/doctor_state.py
+++ b/hermes_cli/doctor_state.py
@@ -263,6 +263,9 @@ def _check_scratch_dir(hermes_home: Path, _DHH: str) -> None:
scratch = get_scratch_dir(hermes_home, prune=False)
size = _human_bytes(scratch_dir_usage_bytes(scratch))
check_ok(f"{_DHH}/cache/scratch/ is the scratch dir (TMPDIR; {size}, entries pruned after {SCRATCH_MAX_IDLE_HOURS}h idle)")
+ from hermes_constants_scratch_stuck import removal_hint, stuck_entries
+ for entry in stuck_entries(scratch):
+ check_warn(f"{_DHH}/cache/scratch/{entry.name}/ holds files you may not delete, so the prune skips it — {removal_hint(entry)}")
for name, nbytes in unpruned_cache_hogs(hermes_home):
check_warn(
f"{_DHH}/cache/{name}/ is {_human_bytes(nbytes)} and outside every pruner "
diff --git a/hermes_constants_scratch.py b/hermes_constants_scratch.py
index 52bc5b94a97..43625927b0b 100644
--- a/hermes_constants_scratch.py
+++ b/hermes_constants_scratch.py
@@ -21,6 +21,8 @@ import time
from logging.handlers import RotatingFileHandler
from pathlib import Path
+import hermes_constants_scratch_stuck as stuck
+
logger = logging.getLogger(__name__)
# How long a TERMed process gets before KILL; browsers exit well within this.
@@ -362,6 +364,10 @@ def _prune_idle_entries(
root: Path, max_idle_hours: float, skip_names: frozenset[str], audit: logging.Logger,
) -> int:
cutoff = time.time() - max_idle_hours * 3600
+ # Entries already known to hold files this user cannot delete are skipped before any
+ # walk, until their fingerprint changes (hermes_constants_scratch_stuck).
+ stuck_state, stuck_names = stuck.load_skips(root)
+ skip_names = skip_names | stuck_names | {stuck.STUCK_STATE_NAME}
try:
entries = [e for e in root.iterdir() if e.name not in skip_names]
except OSError:
@@ -421,9 +427,10 @@ def _prune_idle_entries(
os.fspath(entry),
)
continue
+ denied = False
try:
if entry.is_dir() and not entry.is_symlink():
- shutil.rmtree(entry, ignore_errors=True)
+ denied = stuck.rmtree_collecting_permission_errors(entry)
else:
entry.unlink()
except OSError as exc:
@@ -436,7 +443,10 @@ def _prune_idle_entries(
# above refuses to invent (P2 on #134173).
try:
os.lstat(entry)
- audit.info("scratch prune: could not fully remove %r", os.fspath(entry))
+ if denied and stuck.tracking_supported():
+ stuck.record(root, stuck_state, entry, audit)
+ else:
+ audit.info("scratch prune: could not fully remove %r", os.fspath(entry))
continue
except FileNotFoundError:
pass
diff --git a/hermes_constants_scratch_stuck.py b/hermes_constants_scratch_stuck.py
new file mode 100644
index 00000000000..81e7b016f0f
--- /dev/null
+++ b/hermes_constants_scratch_stuck.py
@@ -0,0 +1,124 @@
+"""Scratch entries the prune cannot delete as this user: remembered once, skipped until they change.
+
+A container that ran as root and wrote into a bind mount under scratch (the Docker terminal
+backend without ``docker_run_as_host_user``, a test or review lane's ``docker run -v``) leaves
+files this user cannot unlink. ``rmtree(ignore_errors=True)`` removes what it can, the entry
+stays idle, and every hourly pass re-walked the whole subtree and logged the same
+"could not fully remove" line again (218 times per entry on one host). Here such an entry is
+recorded with a fingerprint of its top directory, the user gets ONE line naming the command
+that clears it, and later passes skip it before walking it until the fingerprint changes or
+the entry is gone. Nothing is escalated or shelled out to: the hint is for the user to run.
+
+POSIX only: without ``os.getuid`` (Windows) a PermissionError is usually a transient open
+handle, so the prune keeps retrying there exactly as before.
+"""
+from __future__ import annotations
+
+import contextlib
+import errno
+import json
+import os
+import shlex
+import shutil
+import sys
+import time
+from pathlib import Path
+
+STUCK_STATE_NAME = ".prune_stuck.json"
+_PERMISSION_ERRNOS = frozenset({errno.EACCES, errno.EPERM})
+
+
+def tracking_supported() -> bool:
+ return os.name == "posix" and hasattr(os, "getuid")
+
+
+def _fingerprint(entry: Path) -> list[int] | None:
+ try:
+ st = os.lstat(entry)
+ except OSError:
+ return None
+ # ctime too: ``sudo chown -R``/``chmod -R`` fixes the entry without touching its mtime.
+ return [st.st_ino, st.st_mtime_ns, st.st_ctime_ns]
+
+
+def _state_path(root: Path) -> Path:
+ return root / STUCK_STATE_NAME
+
+
+def _read(root: Path) -> dict[str, dict]:
+ try:
+ data = json.loads(_state_path(root).read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError):
+ return {}
+ return {k: v for k, v in data.items() if isinstance(v, dict)} if isinstance(data, dict) else {}
+
+
+def _write(root: Path, state: dict[str, dict]) -> None:
+ path = _state_path(root)
+ with contextlib.suppress(OSError):
+ if not state:
+ path.unlink(missing_ok=True)
+ return
+ tmp = path.with_name(path.name + ".tmp")
+ tmp.write_text(json.dumps(state, indent=1, sort_keys=True), encoding="utf-8")
+ os.replace(tmp, path)
+
+
+def load_skips(root: Path, *, persist: bool = True) -> tuple[dict[str, dict], frozenset[str]]:
+ """``(state, names to skip)``. Records whose entry vanished or whose fingerprint changed
+ (someone touched or partly cleaned it) are dropped, so that entry is judged normally."""
+ if not tracking_supported():
+ return {}, frozenset()
+ state = _read(root)
+ live = {name: rec for name, rec in state.items() if rec.get("fp") == _fingerprint(root / name)}
+ if persist and live != state:
+ _write(root, live)
+ return live, frozenset(live)
+
+
+def rmtree_collecting_permission_errors(entry: Path) -> bool:
+ """``rmtree`` that keeps going past errors (like ``ignore_errors=True``) and returns True
+ when any of them was EACCES/EPERM, i.e. what is left is not removable by this user."""
+ denied = False
+
+ def _onexc(_func, _path, exc):
+ nonlocal denied
+ if isinstance(exc, OSError) and exc.errno in _PERMISSION_ERRNOS:
+ denied = True
+
+ if sys.version_info >= (3, 12):
+ shutil.rmtree(entry, onexc=_onexc)
+ else: # pragma: no cover - 3.11
+ shutil.rmtree(entry, onerror=lambda f, p, info: _onexc(f, p, info[1]))
+ return denied
+
+
+def removal_hint(entry: Path) -> str:
+ hint = f"sudo rm -rf {shlex.quote(os.fspath(entry))}"
+ if shutil.which("docker"):
+ hint += (
+ f" (or without sudo: docker run --rm -v {shlex.quote(os.fspath(entry.parent) + ':/s')}"
+ f" alpine rm -rf {shlex.quote('/s/' + entry.name)})"
+ )
+ return hint
+
+
+def record(root: Path, state: dict[str, dict], entry: Path, audit) -> None:
+ """Remember *entry* as not removable by this user and log the one actionable line."""
+ fp = _fingerprint(entry)
+ if fp is None:
+ return
+ state[entry.name] = {"fp": fp, "first_seen": time.time()}
+ _write(root, state)
+ audit.info(
+ "scratch prune: cannot remove %r: it holds files this user (uid %d) may not delete, "
+ "usually root-owned output of a Docker container; skipping it until it changes. "
+ "Remove it with: %s",
+ os.fspath(entry), os.getuid(), removal_hint(entry), # windows-footgun: ok — only reached when tracking_supported()
+ )
+
+
+def stuck_entries(root: Path) -> list[Path]:
+ """Entries the prune is currently skipping as not removable (for ``hermes doctor``)."""
+ _state, names = load_skips(root, persist=False)
+ return [root / name for name in sorted(names)]
diff --git a/tests/test_scratch_prune_undeletable.py b/tests/test_scratch_prune_undeletable.py
new file mode 100644
index 00000000000..e7f14deda07
--- /dev/null
+++ b/tests/test_scratch_prune_undeletable.py
@@ -0,0 +1,84 @@
+"""An idle scratch entry this user cannot fully delete is reported once and not re-walked hourly."""
+
+import os
+import stat
+import time
+
+import pytest
+
+import hermes_constants_scratch as scratch
+
+pytestmark = pytest.mark.skipif(
+ not hasattr(os, "getuid") or os.getuid() == 0,
+ reason="needs a POSIX non-root user: root deletes through a read-only dir",
+)
+
+OLD = time.time() - 48 * 3600
+
+
+def _undeletable_entry(root):
+ """``lane/ro/blob`` with ``ro`` mode 0500: rmtree removes everything else and fails
+ on ``blob`` with EACCES, the same residue a root-owned container write leaves."""
+ entry = root / "lane"
+ ro = entry / "ro"
+ ro.mkdir(parents=True)
+ (entry / "deletable.txt").write_text("x", encoding="utf-8")
+ (ro / "blob").write_text("left by a container", encoding="utf-8")
+ for p in (ro / "blob", entry / "deletable.txt", ro, entry):
+ os.utime(p, (OLD, OLD))
+ ro.chmod(stat.S_IRUSR | stat.S_IXUSR)
+ return entry, ro
+
+
+def _walk_spy(monkeypatch):
+ walked = []
+ real_bytes, real_state = scratch._tree_bytes, scratch.subtree_touch_state
+ monkeypatch.setattr(scratch, "_tree_bytes", lambda p: walked.append(p) or real_bytes(p))
+ monkeypatch.setattr(scratch, "subtree_touch_state", lambda p, c: walked.append(p) or real_state(p, c))
+ monkeypatch.setattr(scratch, "reap_processes_rooted_in", lambda *_a, **_k: 0)
+ return walked
+
+
+def test_undeletable_entry_logged_once_then_skipped_without_walk(tmp_path, monkeypatch):
+ root = tmp_path / "scratch"
+ entry, ro = _undeletable_entry(root)
+ log_file = tmp_path / "logs" / "scratch-prune.log"
+ walked = _walk_spy(monkeypatch)
+ try:
+ assert scratch.prune_idle_entries(root, 24, frozenset(), log_file) == 0
+ assert entry in walked
+ walked.clear()
+ # A day later: pass 1's partial delete no longer counts as recent activity, so on
+ # main the entry is selected, walked and logged again (and again every hour after).
+ later = time.time() + 48 * 3600
+ monkeypatch.setattr(scratch.time, "time", lambda: later)
+ assert scratch.prune_idle_entries(root, 24, frozenset(), log_file) == 0
+ assert entry not in walked, "a known-undeletable entry must not be re-walked every pass"
+ lines = [ln for ln in log_file.read_text(encoding="utf-8-sig").splitlines() if repr(str(entry)) in ln]
+ assert len(lines) == 1, lines
+ assert "sudo rm -rf" in lines[0] and str(entry) in lines[0], lines[0]
+ assert (ro / "blob").exists()
+ finally:
+ ro.chmod(0o700)
+
+
+def test_changed_or_fixed_entry_is_judged_again(tmp_path, monkeypatch):
+ root = tmp_path / "scratch"
+ entry, ro = _undeletable_entry(root)
+ log_file = tmp_path / "logs" / "scratch-prune.log"
+ walked = _walk_spy(monkeypatch)
+ try:
+ assert scratch.prune_idle_entries(root, 24, frozenset(), log_file) == 0
+ # The one line tells the user how to clear it.
+ assert f"sudo rm -rf {entry}" in log_file.read_text(encoding="utf-8-sig")
+ # The user fixes the permissions; the entry's own mtime changing is what un-skips it.
+ ro.chmod(0o700)
+ os.utime(entry, (OLD + 60, OLD + 60))
+ walked.clear()
+ assert scratch.prune_idle_entries(root, 24, frozenset(), log_file) == 1
+ assert entry in walked
+ assert not entry.exists()
+ assert not (root / ".prune_stuck.json").exists()
+ finally:
+ if ro.exists():
+ ro.chmod(0o700)
diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md
index 6743dca51a9..e839539d41a 100644
--- a/website/docs/user-guide/configuration.md
+++ b/website/docs/user-guide/configuration.md
@@ -270,8 +270,12 @@ background `terminal` process, the gateway, a CLI session, or anything they laun
still their child) is never stopped: an idle entry one of them is working inside is kept
instead. Each entry is re-checked right before it is deleted, so
one written to while the prune was stopping processes is kept. Every removal, stopped
-process and kept entry is recorded in `~/.hermes/logs/scratch-prune.log`. `hermes doctor`
-reports the directory and its size, and
+process and kept entry is recorded in `~/.hermes/logs/scratch-prune.log`. An entry holding
+files your user may not delete (typically root-owned output of a Docker container that wrote
+into a bind mount) is logged once with the command that removes it (`sudo rm -rf '<path>'`, or
+a `docker run … rm -rf` alternative when Docker is installed) and then skipped, not re-scanned
+every hour, until it changes or is gone; Hermes never runs that command itself. `hermes doctor`
+reports the directory, its size and any such skipped entries, and
warns about directories over 1 GB elsewhere under `cache/` that no pruner covers.
`desktop.font_family` sets the font for chat and the rest of the Hermes Desktop interface (the terminal pane has its own key above). Give it one installed family name (for example, `OpenDyslexic` or `Atkinson Hyperlegible`) or a CSS font stack; Hermes keeps the active theme's own stack behind it so CJK and emoji glyphs still resolve, and an empty value uses the theme's font. Edit it in **Settings → Appearance → Chat Font**.