Skip to content

The agent that grows with you

fix(pm): prune abandoned Windows leases under their lock

helix4u662867c5289b
diff --git a/hermes_cli/runtime_state.py b/hermes_cli/runtime_state.py
index c799d86729d..b6d595b2b2e 100644
--- a/hermes_cli/runtime_state.py
+++ b/hermes_cli/runtime_state.py
@@ -213,6 +213,39 @@ def leases_held(generation: Path) -> bool:
return _prune_unlocked_leases(generation / ".leases")
+def _open_lease_probe(lease: Path) -> int:
+ """Let a Windows pruner unlink while still holding its exclusion lock.
+
+ Creator handles deliberately keep their ordinary sharing: an open creator
+ between O_EXCL and locking must still prevent a peer from deleting its lease.
+ """
+ if os.name != "nt":
+ return os.open(lease, os.O_RDWR)
+
+ import ctypes
+ from ctypes import wintypes
+ import msvcrt
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel32.CreateFileW.argtypes = [
+ wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p,
+ wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE,
+ ]
+ kernel32.CreateFileW.restype = wintypes.HANDLE
+ kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
+ kernel32.CloseHandle.restype = wintypes.BOOL
+ # GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | WRITE | DELETE, OPEN_EXISTING.
+ handle = kernel32.CreateFileW(str(lease), 0xC0000000, 0x7, None, 3, 0x80, None)
+ if handle == ctypes.c_void_p(-1).value:
+ raise ctypes.WinError(ctypes.get_last_error())
+ try:
+ # The CRT owns the handle after conversion; os.close releases it and its lock.
+ return msvcrt.open_osfhandle(handle, os.O_RDWR | os.O_BINARY | os.O_NOINHERIT)
+ except BaseException:
+ kernel32.CloseHandle(handle)
+ raise
+
+
def _prune_unlocked_leases(leases: Path) -> bool:
"""Remove abandoned lease files and report whether any live lock remains.
@@ -220,13 +253,12 @@ def _prune_unlocked_leases(leases: Path) -> bool:
``atexit``. Cleaning those unlocked files whenever a reader arrives bounds leaks in
the selected generation too, which generation GC intentionally never visits.
- Fail closed: a lease this user cannot open (root-owned 0o600 from a ``sudo hermes`` on the
- same checkout) is held, never a boot failure.
+ Fail closed: a lease this user cannot open or remove is held, never a boot failure.
"""
held = False
for lease in leases.glob("*"):
try:
- fd = os.open(lease, os.O_RDWR)
+ fd = _open_lease_probe(lease)
except FileNotFoundError:
continue
except OSError:
@@ -236,8 +268,12 @@ def _prune_unlocked_leases(leases: Path) -> bool:
if not _lock(fd, wait=False):
held = True
else:
- with suppress(OSError):
+ try:
lease.unlink()
+ except FileNotFoundError:
+ pass # A peer already removed it.
+ except OSError:
+ held = True
finally:
os.close(fd)
return held
diff --git a/tests/pm/test_runtime_gc.py b/tests/pm/test_runtime_gc.py
index e389cbea7e7..f5b53789956 100644
--- a/tests/pm/test_runtime_gc.py
+++ b/tests/pm/test_runtime_gc.py
@@ -54,6 +54,7 @@ def test_collector_yields_to_an_in_flight_stage(tmp_path):
assert collect_runtime_generations(root) == [aborted]
def test_next_reader_removes_lease_left_by_hard_exit(tmp_path):
from hermes_cli.runtime_state import lease_directory
@@ -83,6 +84,7 @@ os._exit(0)
assert list(leases.iterdir()) == []
@pytest.mark.skipif(getattr(os, "geteuid", lambda: 1)() == 0, reason="root opens mode-0 files")
def test_prune_keeps_unopenable_sibling_lease(tmp_path):
"""A lease this user cannot open (root-owned, `sudo hermes` on the same checkout) is HELD:
@@ -103,6 +105,7 @@ def test_prune_keeps_unopenable_sibling_lease(tmp_path):
foreign.chmod(0o600)
def test_lease_taken_while_a_peer_pruned_it_is_retaken(tmp_path, monkeypatch):
"""A peer's prune can lock-and-unlink our lease between create and flock; the writer must
end up holding a lease the collectors can see, never an unlinked inode."""
diff --git a/tests/pm/test_runtime_lease_cleanup.py b/tests/pm/test_runtime_lease_cleanup.py
new file mode 100644
index 00000000000..5976b41defe
--- /dev/null
+++ b/tests/pm/test_runtime_lease_cleanup.py
@@ -0,0 +1,106 @@
+"""Lease pruning keeps its exclusion through deletion and retains uncertain owners."""
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+from hermes_cli import runtime_state
+
+
+def _generation(root: Path) -> Path:
+ root.mkdir(parents=True)
+ (root / ".lease-managed").touch()
+ (root / ".leases").mkdir()
+ return root
+
+
+def test_pruner_removes_abandoned_file_while_holding_its_lock(tmp_path, monkeypatch):
+ generation = _generation(tmp_path / "generation")
+ lease = generation / ".leases" / "abandoned"
+ lease.touch()
+ unlink = Path.unlink
+ locked_at_delete = []
+
+ def observe_unlink(path, *args, **kwargs):
+ if path == lease:
+ probe = os.open(path, os.O_RDWR)
+ try:
+ locked_at_delete.append(not runtime_state._lock(probe, wait=False))
+ finally:
+ os.close(probe)
+ return unlink(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "unlink", observe_unlink)
+ assert not runtime_state.leases_held(generation)
+ assert not lease.exists()
+ assert locked_at_delete == [True]
+
+
+def test_pruner_keeps_a_windows_creator_before_its_first_lock(tmp_path, monkeypatch):
+ generation = _generation(tmp_path / "generation")
+ lock = runtime_state._lock
+ observed = []
+
+ def interleave_prune(fd, *, wait, **kwargs):
+ if wait and not observed:
+ created = list((generation / ".leases").iterdir())
+ observed.append((runtime_state.leases_held(generation), created))
+ return lock(fd, wait=wait, **kwargs)
+
+ monkeypatch.setattr(runtime_state, "_lock", interleave_prune)
+ release = runtime_state.lease_directory(generation)
+ try:
+ held, created = observed[0]
+ assert held, "An unlocked but still-open creator is not an abandoned owner"
+ assert len(created) == 1 and created[0].exists()
+ assert runtime_state.leases_held(generation)
+ finally:
+ release()
+ assert list((generation / ".leases").iterdir()) == []
+
+
[email protected]("collector", ["runtime", "dependencies"])
+def test_unremovable_lease_keeps_its_generation(tmp_path, monkeypatch, collector):
+ from pm.environments import install_state_dir, runtime_facts_path
+ from pm.runtime import collect_runtime_generations
+
+ if collector == "runtime":
+ root = tmp_path / "runtime"
+ generation = _generation(root / "generations" / "obsolete")
+ (generation / "pm-runtime.json").write_text("{}", encoding="utf-8")
+ collect = lambda: collect_runtime_generations(root)
+ else:
+ project = tmp_path / "project"
+ project.mkdir()
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
+ state = install_state_dir(project)
+ generation = _generation(state / "environments" / "obsolete")
+ selected = _generation(state / "environments" / "selected") / "venv"
+ selected.mkdir()
+ (selected / "pyvenv.cfg").write_text("version = 3.14\n", encoding="utf-8")
+ runtime_facts_path(project).write_text(
+ json.dumps({"packages": {"venv": {"environment": str(selected)}}}),
+ encoding="utf-8",
+ )
+ collect = lambda: runtime_state.collect_generations(project, min_age_seconds=0)
+ lease = generation / ".leases" / "abandoned"
+ lease.touch()
+ unlink = Path.unlink
+
+ def refuse_lease(path, *args, **kwargs):
+ if path == lease:
+ raise PermissionError("Synthetic lease deletion refusal")
+ return unlink(path, *args, **kwargs)
+
+ with monkeypatch.context() as patch:
+ patch.setattr(Path, "unlink", refuse_lease)
+ assert runtime_state.leases_held(generation)
+ assert collect() == []
+ assert generation.is_dir() and lease.exists()
+ assert collect() == [generation]
+ assert not generation.exists()