Skip to content

The agent that grows with you

fix(process): the spawn ledger's file lock waits at most 2 s

kshitijk4poor53a271506e95
diff --git a/hermes_cli/active_sessions.py b/hermes_cli/active_sessions.py
index 7799ac809a9..39da4eae334 100644
--- a/hermes_cli/active_sessions.py
+++ b/hermes_cli/active_sessions.py
@@ -195,27 +195,46 @@ def _lease_paths(
return home / "runtime" / "active_sessions.json", home / "runtime" / "active_sessions.lock"
-def _flock(fh, *, lock: bool) -> None:
- """Exclusive whole-file lock/unlock on ``fh`` (fcntl on POSIX, msvcrt on Windows)."""
+def _flock(fh, *, lock: bool, timeout: Optional[float] = None) -> None:
+ """Exclusive whole-file lock/unlock on ``fh`` (fcntl on POSIX, msvcrt on Windows). With
+ ``timeout`` the lock is polled non-blocking and OSError raised once it passes, so a holder that is
+ stopped mid-write (SIGSTOP, a debugger) cannot stall the caller forever."""
if os.name == "nt":
import msvcrt
fh.seek(0)
- msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if lock else msvcrt.LK_UNLCK, 1)
else:
import fcntl
- fcntl.flock(fh.fileno(), fcntl.LOCK_EX if lock else fcntl.LOCK_UN)
+ if not lock:
+ if os.name == "nt":
+ msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
+ return
+ deadline = None if timeout is None else time.monotonic() + timeout
+ while True:
+ try:
+ if os.name == "nt":
+ msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if deadline is None else msvcrt.LK_NBLCK, 1)
+ else:
+ fcntl.flock(fh.fileno(), fcntl.LOCK_EX if deadline is None else fcntl.LOCK_EX | fcntl.LOCK_NB)
+ return
+ except OSError:
+ if deadline is None or time.monotonic() >= deadline:
+ raise
+ time.sleep(0.02)
class _FileLock:
- def __init__(self, path: Path):
+ def __init__(self, path: Path, *, timeout: Optional[float] = None):
self.path = path
+ self.timeout = timeout
self._fh = None
def __enter__(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
self._fh = open(self.path, "a+b")
try:
- _flock(self._fh, lock=True)
+ _flock(self._fh, lock=True, timeout=self.timeout)
except Exception as exc:
self._fh.close()
self._fh = None
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index cfdd9846e6f..b034f52a73c 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -460,6 +460,11 @@ def _new_entry(
)
+# A ledger writer stopped mid-write (SIGSTOP, a debugger) must not stall every spawn_owned on the
+# machine: past this the write goes ahead under the thread lock alone, as when no lock can be taken.
+_LEDGER_LOCK_TIMEOUT_S = 2.0
+
+
def _append_entry(entry: LedgerEntry) -> bool:
"""Prune dead entries and append ``entry`` — the ONLY ledger write path.
@@ -474,7 +479,7 @@ def _append_entry(entry: LedgerEntry) -> bool:
path = _ledger_path()
with _LEDGER_LOCK, ExitStack() as held:
try:
- held.enter_context(_FileLock(path.with_name(path.name + ".lock")))
+ held.enter_context(_FileLock(path.with_name(path.name + ".lock"), timeout=_LEDGER_LOCK_TIMEOUT_S))
except (RuntimeError, OSError):
# Best-effort like the write itself: the thread lock is then the only guard.
logger.debug("spawn ledger lock unavailable", exc_info=True)
diff --git a/tests/hermes_cli/test_process_identity.py b/tests/hermes_cli/test_process_identity.py
index 909c2fd2353..b04ded41107 100644
--- a/tests/hermes_cli/test_process_identity.py
+++ b/tests/hermes_cli/test_process_identity.py
@@ -491,3 +491,24 @@ def test_concurrent_processes_never_drop_each_others_ledger_rows(tmp_path):
for w in writers:
w.stdin.close()
w.wait(timeout=30)
+
+
[email protected](os.name == "nt", reason="fcntl lock holder")
+def test_a_stalled_ledger_lock_holder_never_blocks_a_spawn(tmp_path, monkeypatch):
+ """A holder stopped mid-write (SIGSTOP, a debugger) kept the lock: every spawn_owned on the
+ machine then waited forever. Past the timeout the row is written under the thread lock alone."""
+ import fcntl
+ import threading
+
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
+ monkeypatch.setattr(pi, "_LEDGER_LOCK_TIMEOUT_S", 0.3)
+ ledger = pi._ledger_path()
+ ledger.parent.mkdir(parents=True, exist_ok=True)
+ with open(ledger.with_name(ledger.name + ".lock"), "a+b") as holder:
+ fcntl.flock(holder.fileno(), fcntl.LOCK_EX) # the stalled holder
+ done = []
+ writer = threading.Thread(target=lambda: done.append(pi.register_child(os.getpid(), "stalled-lock")),
+ daemon=True)
+ writer.start()
+ writer.join(timeout=10)
+ assert done == [True], "register_child waited on a stalled lock holder"