The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(pm): a launcher waiting on a reclaimed state's install lock retries on the live file
Tekniumd9bdc4961237
diff --git a/hermes_cli/runtime_state.py b/hermes_cli/runtime_state.pyindex b6d595b2b2e..7eba6e79f6c 100644--- a/hermes_cli/runtime_state.py+++ b/hermes_cli/runtime_state.py@@ -22,6 +22,7 @@ from pm.filesystem import (durable_write_bytes as _atomic_bytes,file_digest as _digest,lock_fd as _lock,+ lock_path,read_bytes_or_none as _bytes,)@@ -44,16 +45,13 @@ def runtime_lock(project: Path, *, timeout: float | None = INSTALL_LOCK_TIMEOUT_(``activate_dependencies`` still selects and leases the committed generation), writers thatcannot be skipped pass ``timeout=None`` — an install the user asked for is theirs to wait on."""- state = install_state_dir(project)- state.mkdir(parents=True, exist_ok=True)- fd = os.open(state / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600)+ lock = install_state_dir(project) / ".install.lock"+ fd = lock_path(lock, timeout=timeout, create_dir=True)+ if fd is None:+ LOG.warning("dependency lock still held after %ss; continuing without it (%s)", timeout, lock)+ yield False+ returntry:- if not _lock(fd, wait=True, timeout=timeout):- LOG.warning(- "dependency lock still held after %ss; continuing without it (%s)",- timeout, state / ".install.lock")- yield False- returnyield Truefinally:os.close(fd)diff --git a/pm/filesystem.py b/pm/filesystem.pyindex ab0b4bb87ef..6f135ccd727 100644--- a/pm/filesystem.py+++ b/pm/filesystem.py@@ -107,6 +107,47 @@ def unlock_fd(fd: int) -> None:fcntl.flock(fd, fcntl.LOCK_UN)+def lock_path(path: Path, *, timeout: float | None, create_dir: bool = False) -> int | None:+ """Open *path* (created if missing), wait up to ``timeout`` for its byte lock and return the+ held descriptor; None when the wait expired. The caller closes the descriptor to release.++ A holder that removes the lock file unlinks it while still holding it (the install-state+ collector), so a waiter that opened the old file gets a lock on an inode no one else can+ reach while the next opener creates a fresh file: both would hold "the" lock. Once held,+ the descriptor must still be the file at *path*, else retry on the current one. Windows+ cannot unlink an open file, so the check is POSIX-only. ``create_dir`` re-creates the+ parent the remover may have taken with it.+ """+ deadline = None if timeout is None else time.monotonic() + timeout+ while True:+ if create_dir:+ path.parent.mkdir(parents=True, exist_ok=True)+ try:+ fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)+ except FileNotFoundError:+ if create_dir:+ continue # parent removed between mkdir and open+ raise+ try:+ remaining = None if deadline is None else max(0.0, deadline - time.monotonic())+ if not lock_fd(fd, wait=True, timeout=remaining):+ os.close(fd)+ return None+ if os.name == "nt" or _same_file(fd, path):+ return fd+ except BaseException:+ os.close(fd)+ raise+ os.close(fd)+++def _same_file(fd: int, path: Path) -> bool:+ try:+ return os.path.samestat(os.fstat(fd), os.stat(path))+ except FileNotFoundError:+ return False++def read_bytes_or_none(path: Path) -> bytes | None:try:return path.read_bytes()diff --git a/pm/install_states.py b/pm/install_states.pyindex c62b1803881..9b3fb38a4f5 100644--- a/pm/install_states.py+++ b/pm/install_states.py@@ -100,17 +100,17 @@ def orphan_install_states(installs: Path) -> list[Path]:def _install_lock(state: Path):"""Hold the state's install lock (``runtime_lock``'s) without waiting: yields False whileanother process holds it, or when it cannot even be opened."""- from pm.filesystem import lock_fd+ from pm.filesystem import lock_pathtry:- fd = os.open(state / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600)+ fd = lock_path(state / ".install.lock", timeout=0)except OSError:fd = Noneif fd is None:yield Falsereturntry:- yield lock_fd(fd, wait=False)+ yield Truefinally:os.close(fd)@@ -165,13 +165,20 @@ def collect_orphan_install_states(installs: Path) -> list[Path]:emptied = _remove_contents(state)if emptied: # still under the lock: only the record is left, so it goes lastshutil.rmtree(state / "inputs", ignore_errors=True)+ # POSIX: unlink while still holding it. A launcher already waiting on this file+ # then sees it gone once it gets the lock and retries on a fresh one (lock_path);+ # unlinked after release, the path could be a file that launcher just made.+ if os.name != "nt":+ with contextlib.suppress(OSError):+ (state / ".install.lock").unlink()if not emptied:logger.debug("orphan install state %s only partly removed; retried next pass", state.name)continue# Non-recursive: a launch from a checkout recreated at the same path may have taken the# lock the moment it was released and written here; its files make rmdir fail, not vanish.with contextlib.suppress(OSError):- (state / ".install.lock").unlink()+ if os.name == "nt": # an open file cannot be unlinked: a waiting launcher keeps it+ (state / ".install.lock").unlink()state.rmdir()if not state.exists():removed.append(state)diff --git a/tests/pm/test_install_lock_reclaim_race.py b/tests/pm/test_install_lock_reclaim_race.pynew file mode 100644index 00000000000..8382ac08605--- /dev/null+++ b/tests/pm/test_install_lock_reclaim_race.py@@ -0,0 +1,65 @@+"""Reclaiming an orphan install state never leaves two holders of its install lock+(``pm/install_states.py``, ``hermes_cli.runtime_state.runtime_lock``)."""++from __future__ import annotations++import os+import threading++import pm.environments+import pm.install_states+import pytest+from hermes_cli.runtime_state import runtime_lock+from pm.environments import install_key+from pm.install_states import collect_orphan_install_states++[email protected]("posix")+def test_launcher_waiting_on_a_reclaimed_state_lock_excludes_later_launchers(tmp_path, monkeypatch):+ """A launcher from a checkout re-created at the recorded path opens the lock while the+ collector holds it, then gets it when the collector lets go. The lock file it waited on+ is the one the collector removes; whatever it ends up holding must still exclude the+ next launcher, or two installers write one state."""+ installs = tmp_path / "home" / "installs"+ checkout = tmp_path / "home" / "cache" / "scratch" / "clone" # provably deleted: reclaimed+ state = installs / install_key(checkout)+ (state / "inputs").mkdir(parents=True)+ (state / "inputs" / ".project-root").write_text(str(checkout), encoding="utf-8")+ (state / "environments" / "gen" / "venv").mkdir(parents=True)+ monkeypatch.setattr(pm.environments, "installs_root", lambda: installs)++ opened, held, done = threading.Event(), threading.Event(), threading.Event()+ real_open = os.open++ def observed_open(path, *args, **kwargs):+ fd = real_open(path, *args, **kwargs)+ if threading.current_thread().name == "launcher-a" and os.fspath(path).endswith(".install.lock"):+ opened.set()+ return fd++ def launcher_a():+ with runtime_lock(checkout, timeout=None) as got:+ if got:+ held.set()+ done.wait(30)++ a = threading.Thread(target=launcher_a, name="launcher-a", daemon=True)+ real_leased = pm.install_states._leased++ def checkout_recreated_mid_reclaim(path):+ # Called under the collector's lock, after its orphan re-check.+ checkout.mkdir(parents=True)+ a.start()+ assert opened.wait(30)+ return real_leased(path)++ monkeypatch.setattr(os, "open", observed_open)+ monkeypatch.setattr(pm.install_states, "_leased", checkout_recreated_mid_reclaim)+ try:+ collect_orphan_install_states(installs)+ assert held.wait(30)+ with runtime_lock(checkout, timeout=0) as launcher_b:+ assert launcher_b is False, "two launchers hold the same state's install lock"+ finally:+ done.set()+ a.join(30)