Skip to content

The agent that grows with you

fix(update): a checkout-lock probe never reads another probe as a running update

Tekniuma4b2ba62dcb0
diff --git a/hermes_cli/update_lock.py b/hermes_cli/update_lock.py
index 62d4a159e97..a0bfe9c9de5 100644
--- a/hermes_cli/update_lock.py
+++ b/hermes_cli/update_lock.py
@@ -979,7 +979,15 @@ _HELD: dict | None = None
_JOBS: list = []
-def _try_lock(fd: int) -> bool:
+def _try_lock(fd: int, *, shared: bool = False) -> bool:
+ """Lock the owner byte and every lease byte without waiting; False when any is held.
+
+ ``shared`` is the held? probe (:func:`checkout_lock_held`): holders lock exclusively, so a
+ shared try fails only on a holder and two probes never read each other as one. POSIX drops it
+ when the caller closes ``fd``; Windows gives it straight back.
+ """
+ if shared:
+ return _try_shared_lock(fd)
if sys.platform == "win32":
if not _lock_bytes(fd, _WINDOWS_LOCK_OFFSET):
return False
@@ -997,6 +1005,40 @@ def _try_lock(fd: int) -> bool:
return True
+def _try_shared_lock(fd: int) -> bool:
+ if sys.platform == "win32":
+ # msvcrt has no shared mode: LockFileEx without LOCKFILE_EXCLUSIVE_LOCK over the owner byte
+ # and the 16 lease bytes. Any failure answers held, as the exclusive probe did.
+ import ctypes
+ import msvcrt
+ from ctypes import wintypes
+
+ class _Overlapped(ctypes.Structure):
+ _fields_ = [("Internal", ctypes.c_void_p), ("InternalHigh", ctypes.c_void_p),
+ ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD), ("hEvent", wintypes.HANDLE)]
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel32.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD,
+ wintypes.DWORD, ctypes.POINTER(_Overlapped)]
+ kernel32.LockFileEx.restype = wintypes.BOOL
+ kernel32.UnlockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD,
+ ctypes.POINTER(_Overlapped)]
+ kernel32.UnlockFileEx.restype = wintypes.BOOL
+ handle = wintypes.HANDLE(msvcrt.get_osfhandle(fd))
+ span = 1 + _LEASE_SLOTS
+ if not kernel32.LockFileEx(handle, 0x1, 0, span, 0, ctypes.byref(_Overlapped(Offset=_WINDOWS_LOCK_OFFSET))):
+ return False # 0x1 = LOCKFILE_FAIL_IMMEDIATELY
+ kernel32.UnlockFileEx(handle, 0, span, 0, ctypes.byref(_Overlapped(Offset=_WINDOWS_LOCK_OFFSET)))
+ return True
+ import fcntl
+
+ try:
+ fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
+ except BlockingIOError:
+ return False
+ return True
+
+
def _lock_bytes(fd: int, offset: int, count: int = 1) -> bool:
"""Windows: lock ``count`` bytes at ``offset`` without waiting (fails if any is held)."""
import msvcrt
@@ -1326,9 +1368,12 @@ def update_in_progress(install_root: Path | str | None = None) -> bool:
def checkout_lock_held(install_root: Path | str | None = None) -> bool:
"""True while some process (this one included) holds the checkout kernel lock.
- A probe takes the lock for the microseconds of one try and drops it (closing the fd), the
- way ``marker.sh::checkout_lock_held`` does; an updater acquiring at that instant waits
- :data:`CHECKOUT_CONTENTION_WAIT_SECONDS` instead of failing (R6/D17).
+ A probe takes a SHARED lock on the owner byte and every lease byte for the microseconds of one
+ try and drops it (closing the fd). Holders lock exclusively, so the probe fails only on a real
+ holder, while two probes never read each other as one: a client polling ``ensure`` and the
+ gateway it just started both probe while a killed update's dead marker is still there. An
+ updater acquiring at that instant waits :data:`CHECKOUT_CONTENTION_WAIT_SECONDS` instead of
+ failing (R6/D17).
Only a missing lock file is free. One that exists but cannot be opened answers held, the same
verdict ``marker.sh`` / ``marker.ps1`` and the Desktop probes give, so no reader admits a
@@ -1345,10 +1390,7 @@ def checkout_lock_held(install_root: Path | str | None = None) -> bool:
except OSError:
return True
try:
- if not _try_lock(fd):
- return True
- _unlock(fd)
- return False
+ return not _try_lock(fd, shared=True)
except OSError:
return False
finally:
diff --git a/tests/hermes_cli/test_update_lock_probe_readers.py b/tests/hermes_cli/test_update_lock_probe_readers.py
new file mode 100644
index 00000000000..128d7079e24
--- /dev/null
+++ b/tests/hermes_cli/test_update_lock_probe_readers.py
@@ -0,0 +1,101 @@
+"""The update fence never reads another reader's probe of the checkout lock as a held lock.
+
+A killed update always leaves a dead ``.hermes-update-in-progress`` marker, and while it is there
+``gateway_runtime._update_fenced`` probes the checkout kernel lock on every ``ensure`` poll. The
+gateway that ``ensure`` just started probes the same lock while it boots (its launch reclaims the
+dead marker). A probe that took the lock exclusively turned that collision into "an update owns
+the install": the first launch after the kill refused with ``Gateway draining: update_paused``.
+
+Real processes and the real lock file; a real holder still fences (control).
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+from hermes_cli import gateway_runtime, update_lock
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+_READER = r"""
+import sys, time
+sys.path.insert(0, sys.argv[1])
+from hermes_cli import update_lock
+root, seconds = sys.argv[2], float(sys.argv[3])
+print("ready", flush=True)
+end = time.monotonic() + seconds
+while time.monotonic() < end:
+ update_lock.checkout_lock_held(root)
+"""
+
+_HOLDER = r"""
+import sys
+from pathlib import Path
+sys.path.insert(0, sys.argv[1])
+from hermes_cli import update_lock
+assert update_lock._acquire_checkout(Path(sys.argv[2])) is None
+print("held", flush=True)
+sys.stdin.readline()
+"""
+
+
+def _spawn(code: str, root: Path, *args: str) -> subprocess.Popen:
+ return subprocess.Popen([sys.executable, "-c", code, str(REPO_ROOT), str(root), *args],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, cwd=REPO_ROOT)
+
+
+def install(tmp_path, monkeypatch):
+ """A checkout whose lock file exists, and a home holding a killed update's dead marker."""
+ root = tmp_path / "install"
+ (root / ".git").mkdir(parents=True)
+ (root / ".git" / update_lock.GIT_CHECKOUT_LOCK_NAME).write_bytes(b"")
+ home = tmp_path / "home"
+ home.mkdir()
+ # pid 0 is never a live process: the marker a killed update leaves behind.
+ (home / update_lock.MARKER_NAME).write_text(f"0\n{int(time.time())}\nct:1.000\n", encoding="utf-8")
+ # _update_fenced probes the running checkout's lock (no install_root): make that this one.
+ named = update_lock.checkout_lock_path
+ monkeypatch.setattr(update_lock, "checkout_lock_path", lambda install_root=None: named(install_root or root))
+ return root, home
+
+
+def _assert_readers_never_fence(root: Path, home: Path) -> None:
+ readers = [_spawn(_READER, root, "6") for _ in range(2)]
+ try:
+ for reader in readers:
+ assert reader.stdout.readline().strip() == "ready"
+ fenced = polls = 0
+ end = time.monotonic() + 2.0
+ while time.monotonic() < end:
+ polls += 1
+ fenced += gateway_runtime._update_fenced({home})
+ finally:
+ for reader in readers:
+ reader.kill()
+ reader.wait(timeout=30)
+ assert fenced == 0, f"{fenced} of {polls} ensure polls read another reader's probe as an update"
+
+ holder = _spawn(_HOLDER, root)
+ try:
+ assert holder.stdout.readline().strip() == "held"
+ assert gateway_runtime._update_fenced({home}), "a live holder of the checkout lock must fence"
+ finally:
+ holder.stdin.close()
+ holder.wait(timeout=30)
+ assert not gateway_runtime._update_fenced({home})
+
+
+def test_concurrent_lock_probes_never_fence_a_launch_posix(install) -> None:
+ _assert_readers_never_fence(*install)
+
+
+def test_concurrent_lock_probes_never_fence_a_launch_windows(install) -> None:
+ _assert_readers_never_fence(*install)