Skip to content

The agent that grows with you

fix(pm): orphan-state retention comes from the store owner, and each removal re-checks under the install lock

Tekniumcdbcbe86eaab
diff --git a/pm/install_states.py b/pm/install_states.py
index 2e2b85f2581..ef911788319 100644
--- a/pm/install_states.py
+++ b/pm/install_states.py
@@ -56,54 +56,69 @@ def _last_used(state: Path) -> float:
return max(mtimes)
-def _retention_seconds() -> float | None:
+def _retention_seconds(data_root: Path) -> float | None:
+ """The store-wide retention, from the config of the data root that owns ``installs/``.
+ Every profile of that root shares the store, so the active profile's config must not decide
+ for the others: a named profile without the key would apply the default to a state the root
+ opted out of reclaiming."""
from hermes_cli.config import cfg_get, load_config_readonly
+ from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- days = cfg_get(load_config_readonly(), "worktrees", "install_state_retention_days", default=14)
+ token = set_hermes_home_override(data_root)
+ try:
+ days = cfg_get(load_config_readonly(), "worktrees", "install_state_retention_days", default=14)
+ finally:
+ reset_hermes_home_override(token)
return days * 86400 if isinstance(days, (int, float)) and days > 0 else None
+def _is_orphan(state: Path, data_root: Path, retention: float | None) -> bool:
+ record = state / "inputs" / ".project-root"
+ if not state.is_dir() or not record.is_file():
+ return False
+ try:
+ root = record.read_text(encoding="utf-8-sig").strip()
+ except OSError:
+ return False
+ if not root or Path(root).exists():
+ return False
+ return _provably_deleted(Path(root), data_root) or (
+ retention is not None and time.time() - _last_used(state) > retention)
+
+
def orphan_install_states(installs: Path) -> list[Path]:
"""State dirs whose recorded checkout is provably deleted, or missing and unused past the
retention period. Dirs without a record, and dirs whose checkout exists, are left alone."""
if not installs.is_dir():
return []
data_root = installs.resolve().parent
- retention = _retention_seconds()
- orphans: list[Path] = []
- for state in sorted(installs.iterdir()):
- record = state / "inputs" / ".project-root"
- if not state.is_dir() or not record.is_file():
- continue
- try:
- root = record.read_text(encoding="utf-8-sig").strip()
- except OSError:
- continue
- if not root or Path(root).exists():
- continue
- if _provably_deleted(Path(root), data_root) or (
- retention is not None and time.time() - _last_used(state) > retention):
- orphans.append(state)
- return orphans
+ retention = _retention_seconds(data_root)
+ return [state for state in sorted(installs.iterdir()) if _is_orphan(state, data_root, retention)]
-def _held(state: Path) -> bool:
- """True while a process may still be using this state dir: the install lock is taken or a
- generation lease is held. Any lock we cannot even open counts as held."""
- from hermes_cli.runtime_state import leases_held
+def _install_lock(state: Path):
+ """Hold the state's install lock (``runtime_lock``'s) without waiting: yields False while
+ another process holds it, or when it cannot even be opened."""
from pm.filesystem import lock_fd
- lock = state / ".install.lock"
- if lock.exists():
- try:
- fd = os.open(lock, os.O_RDWR)
- except OSError:
- return True
- try:
- if not lock_fd(fd, wait=False):
- return True
- finally:
- os.close(fd)
+ try:
+ fd = os.open(state / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600)
+ except OSError:
+ fd = None
+ if fd is None:
+ yield False
+ return
+ try:
+ yield lock_fd(fd, wait=False)
+ finally:
+ os.close(fd)
+
+
+def _leased(state: Path) -> bool:
+ """True while a generation lease (dependency env or PM runtime) is held."""
+ from hermes_cli.runtime_state import leases_held
+
for generations in (state / "environments", state / "pm-runtime" / "generations"):
if generations.is_dir():
for generation in generations.iterdir():
@@ -112,25 +127,43 @@ def _held(state: Path) -> bool:
return False
+_KEPT_UNTIL_LAST = frozenset({"inputs", ".install.lock"})
+
+
+def _remove_contents(state: Path) -> bool:
+ """Remove everything but the record and the held lock; True when nothing else is left.
+ The record goes last, and only once everything else is gone: the startup prune runs on a
+ daemon thread and a removal can fail part-way (an open file on Windows, a read-only dir), and
+ a dir that lost its record first would never be recognised again."""
+ for child in state.iterdir():
+ if child.name in _KEPT_UNTIL_LAST:
+ continue
+ if child.is_dir() and not child.is_symlink():
+ shutil.rmtree(child, ignore_errors=True)
+ else:
+ with contextlib.suppress(OSError):
+ child.unlink()
+ return not any(child.name not in _KEPT_UNTIL_LAST for child in state.iterdir())
+
+
def collect_orphan_install_states(installs: Path) -> list[Path]:
- """Remove every unheld orphan state dir under *installs*; returns what was removed."""
+ """Remove every unheld orphan state dir under *installs*; returns what was removed.
+
+ Each candidate is re-checked under its install lock right before removal: removing a large
+ orphan takes long enough for another state to be synced or launched (lease taken and
+ released) since the candidates were listed, and that use must keep it."""
+ if not installs.is_dir():
+ return []
+ data_root = installs.resolve().parent
+ retention = _retention_seconds(data_root)
removed: list[Path] = []
- for state in orphan_install_states(installs):
- if _held(state):
- logger.debug("orphan install state %s is still held; skipped", state.name)
- continue
- # The record goes last, and only once everything else is gone: the startup prune runs on
- # a daemon thread and a removal can fail part-way (an open file on Windows, a read-only
- # dir), and a dir that lost its record first would never be recognised again.
- for child in state.iterdir():
- if child.name == "inputs":
+ for state in [s for s in sorted(installs.iterdir()) if _is_orphan(s, data_root, retention)]:
+ with _install_lock(state) as locked:
+ if not locked or not _is_orphan(state, data_root, retention) or _leased(state):
+ logger.debug("orphan install state %s is held or back in use; skipped", state.name)
continue
- if child.is_dir() and not child.is_symlink():
- shutil.rmtree(child, ignore_errors=True)
- else:
- with contextlib.suppress(OSError):
- child.unlink()
- if any(child.name != "inputs" for child in state.iterdir()):
+ emptied = _remove_contents(state)
+ if not emptied:
logger.debug("orphan install state %s only partly removed; retried next pass", state.name)
continue
shutil.rmtree(state, ignore_errors=True)
diff --git a/tests/pm/test_install_states_gc.py b/tests/pm/test_install_states_gc.py
index adb346094bb..0382af4ac5b 100644
--- a/tests/pm/test_install_states_gc.py
+++ b/tests/pm/test_install_states_gc.py
@@ -113,10 +113,9 @@ def test_unprovable_missing_checkout_is_reclaimed_once_unused_past_retention(tmp
def test_retention_days_from_config_yaml_decides_reclaim(tmp_path, days, reclaimed):
"""0 keeps an unprovable state forever; a custom value moves the cutoff (2 days idle: kept
under the 14-day default, reclaimed under 1)."""
- from hermes_constants import get_hermes_home
-
if days is not None:
- (get_hermes_home() / "config.yaml").write_text(
+ (tmp_path / "home").mkdir()
+ (tmp_path / "home" / "config.yaml").write_text(
f"worktrees:\n install_state_retention_days: {days}\n", encoding="utf-8")
installs = tmp_path / "home" / "installs"
state = _state(installs, "aaaa", tmp_path / "review" / "gone")
@@ -125,3 +124,58 @@ def test_retention_days_from_config_yaml_decides_reclaim(tmp_path, days, reclaim
os.utime(path, (two_days_ago, two_days_ago), follow_symlinks=False)
assert (collect_orphan_install_states(installs) == [state]) is reclaimed
assert state.exists() is not reclaimed
+
+
+def _age(state, days):
+ then = time.time() - days * 86400
+ for path in sorted(state.rglob("*"), reverse=True) + [state]:
+ os.utime(path, (then, then), follow_symlinks=False)
+
+
[email protected](("root_days", "profile_days", "reclaimed"), [(0, None, False), (None, 0, True)])
+def test_retention_comes_from_the_store_owner_not_the_active_profile(tmp_path, root_days, profile_days, reclaimed):
+ """Every profile of a root shares ``<root>/installs``: the root's setting decides for all of
+ them, whichever profile runs the collection."""
+ from hermes_constants import reset_hermes_home_override, set_hermes_home_override
+
+ root = tmp_path / ".hermes"
+ named = root / "profiles" / "work"
+ named.mkdir(parents=True)
+ for home, days in ((root, root_days), (named, profile_days)):
+ if days is not None:
+ (home / "config.yaml").write_text(
+ f"worktrees:\n install_state_retention_days: {days}\n", encoding="utf-8")
+ token = set_hermes_home_override(named)
+ try:
+ installs = pm.environments.installs_root()
+ assert installs == root / "installs"
+ state = _state(installs, "aaaa", tmp_path / "unmounted" / "checkout")
+ _age(state, 30)
+ assert (collect_orphan_install_states(installs) == [state]) is reclaimed
+ finally:
+ reset_hermes_home_override(token)
+ assert state.exists() is not reclaimed
+
+
+def test_state_used_while_another_orphan_is_removed_is_kept(tmp_path, monkeypatch):
+ """Candidates are listed before any removal starts; a state launched (lease taken and
+ released) while an earlier, large orphan is still being removed is back in use."""
+ import pm.install_states
+ from hermes_cli.runtime_state import lease_directory
+
+ installs = tmp_path / "home" / "installs"
+ deleted = _state(installs, "aaaa", tmp_path / "home" / "cache" / "scratch" / "gone")
+ used = _state(installs, "bbbb", tmp_path / "unmounted" / "checkout")
+ (used / "environments" / "gen" / ".lease-managed").touch()
+ _age(used, 30)
+ rmtree = pm.install_states.shutil.rmtree
+
+ def remove_then_launch(path, *args, **kwargs):
+ if path == deleted / "environments":
+ release = lease_directory(used / "environments" / "gen")
+ release()
+ return rmtree(path, *args, **kwargs)
+
+ monkeypatch.setattr(pm.install_states.shutil, "rmtree", remove_then_launch)
+ assert collect_orphan_install_states(installs) == [deleted]
+ assert (used / "environments" / "gen" / "venv").is_dir()
diff --git a/website/docs/user-guide/git-worktrees.md b/website/docs/user-guide/git-worktrees.md
index 08d0bab6b28..96361e49201 100644
--- a/website/docs/user-guide/git-worktrees.md
+++ b/website/docs/user-guide/git-worktrees.md
@@ -259,7 +259,8 @@ worktrees:
archive_retention_days: 30 # 0 keeps archives forever
max_total_gb: 0 # optional budget for idle trees; oldest are archived + removed first
install_state_retention_days: 14 # dependency state of a missing checkout outside .worktrees/ and
- # the data root, unused this long, is reclaimed; 0 keeps it forever
+ # the data root, unused this long, is reclaimed; 0 keeps it forever.
+ # Read from the root profile's config: its profiles share installs/
```
There is no limit on how many trees may exist: hundreds of parallel subagents