The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Nonblocking, all-or-nothing reservations of canonical profile homes.
A held lock inode is never removed: unlinking it creates a second owner. The one exception is a
stale inode this user cannot write (left by a gateway that ran as root, #42685) that a flock through
a read-only descriptor proves unheld; it is replaced once, and every winner checks that the path
still names the inode it locked.
"""
from __future__ import annotations
from contextlib import contextmanager
import json
import logging
import os
from pathlib import Path
import stat
import threading
import time
from hermes_dir_acl import make_user_dir
logger = logging.getLogger(__name__)
class OwnershipConflict(RuntimeError):
pass
class ReservationCancelled(OwnershipConflict):
"""The waiting reserve's caller gave up: nothing reserved, nothing left held."""
def canonical_home(home: Path) -> Path:
return Path(os.path.normcase(str(Path(home).expanduser().resolve())))
def tighten_lock_mode(fd: int) -> None:
"""Validate an open ``gateway.lock`` descriptor and narrow an upgraded lock to 0600.
Main's ``acquire_gateway_runtime_lock`` created the inode with ``open(path, "a+")`` under the
umask (0644 on most hosts), and lock inodes are never replaced, so an upgraded home keeps that
mode forever and discovery refuses it. Only a regular, single-link inode this user owns is
chmodded, through the already-open NOFOLLOW descriptor (no path race); anything else is refused.
"""
info = os.fstat(fd)
if os.name == 'nt':
if not stat.S_ISREG(info.st_mode):
raise PermissionError('unsafe gateway lock owner or type')
return
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid(): # windows-footgun: ok — POSIX branch
raise PermissionError('unsafe gateway lock owner or type')
if stat.S_IMODE(info.st_mode) & 0o077:
if info.st_nlink != 1:
raise PermissionError('unsafe gateway lock link count')
os.fchmod(fd, 0o600) # windows-footgun: ok — POSIX branch
_LOCK_FLAGS = os.O_RDWR | os.O_CREAT | getattr(os, 'O_NOFOLLOW', 0)
def _open_checked(path: Path, flags: int) -> int:
fd = os.open(path, flags, 0o600)
try:
tighten_lock_mode(fd)
except BaseException:
os.close(fd)
raise
return fd
def _unlink_stale_lock(path: Path) -> bool:
"""Unlink a ``gateway.lock`` this user cannot write (another uid's) once a flock taken through a
read-only descriptor proves it unheld. Symlinks, hardlinks, non-regular files, a home whose
``gateway.pid`` names a live process and anything not provably unheld are left in place (False).
An inode this user cannot even open is refused with the recovery command: an unheld snapshot
(``/proc/locks``) does not stop a holder locking it between the read and the unlink."""
from gateway.status import _live_pid_from_record, _read_pid_record
if os.name == 'nt':
return False # a sharing violation is a live holder there
info = os.lstat(path)
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
return False
if _live_pid_from_record(_read_pid_record(path.with_name('gateway.pid'))) is not None:
return False
try:
probe = os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | os.O_NONBLOCK)
except PermissionError:
raise PermissionError(foreign_lock_recovery(path)) from None
import fcntl
try:
if not _same_inode(os.fstat(probe), info):
return False
try:
fcntl.flock(probe, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
return False
# Unlinked while still locked: a late locker of the old inode never shares our lock.
return _unlink_if_same(path, info)
finally:
os.close(probe)
def foreign_lock_recovery(path: Path) -> str:
return (f'foreign_stale_lock: {path} is not readable by this user (left by a gateway run as root?) '
f'and cannot be proven unheld. If `sudo lsof {path}` lists no process, run `sudo rm {path}`.')
def unreadable_lock_recovery(path: Path) -> str | None:
"""The recovery a client prints for a ``gateway.lock`` it cannot even open read-only."""
try:
os.close(os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | os.O_NONBLOCK))
except PermissionError:
return foreign_lock_recovery(path)
except OSError:
pass
return None
def _unlink_if_same(path: Path, info) -> bool:
if not _same_inode(os.lstat(path), info):
return False
os.unlink(path)
return True
def _same_inode(left, right) -> bool:
return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino)
def _open_lock(path: Path) -> int:
"""Descriptor for *path*, replacing a provably stale inode this user cannot use once."""
try:
return _open_checked(path, _LOCK_FLAGS)
except PermissionError:
if not _unlink_stale_lock(path):
raise
# A racer may create the fresh inode first; both then contend for its flock as usual.
return _open_checked(path, _LOCK_FLAGS)
def _still_names(path: Path, fd: int) -> bool:
"""True when *path* is still the inode *fd* locked (a stale-lock replacement unlinks it)."""
if os.name == 'nt':
return True
try:
return _same_inode(os.lstat(path), os.fstat(fd))
except FileNotFoundError:
return False
class ProfileOwnership:
def __init__(self):
self._handles: dict[Path, object] = {}
self._writers: list[threading.Thread] = []
self._mutex = threading.RLock()
@property
def homes(self) -> tuple[Path, ...]:
with self._mutex:
return tuple(self._handles)
def reserve(self, homes, *, purpose: str | None = None) -> None:
"""``purpose`` marks a maintenance reservation in the lock record, so a contender can tell
a short maintenance holder from a gateway runtime owner (:func:`live_runtime_owner`)."""
from gateway.status import _try_acquire_file_lock, _build_pid_record
with self._mutex:
added = []
try:
for home in sorted({canonical_home(p) for p in homes}):
if home in self._handles:
continue
make_user_dir(home, mode=0o700, parents=True, exist_ok=True)
path = home / 'gateway.lock'
fd = _open_lock(path)
handle = os.fdopen(fd, 'r+', encoding='utf-8') # windows-footgun: ok — write-only
try:
if not _try_acquire_file_lock(handle) or not _still_names(path, handle.fileno()):
raise OwnershipConflict(f'Gateway runtime already owns profile {home}')
record = {**_build_pid_record(), 'hermes_home': str(home)}
if purpose:
record['purpose'] = purpose
handle.seek(0)
handle.truncate()
json.dump(record, handle)
handle.flush()
os.fsync(handle.fileno())
except BaseException:
handle.close()
raise
self._handles[home] = handle
added.append(home)
except BaseException:
for home in reversed(added):
self.release(home)
raise
def owns(self, home: Path) -> bool:
with self._mutex:
return canonical_home(home) in self._handles
def release(self, home: Path) -> None:
from gateway.status import _release_file_lock
with self._mutex:
handle = self._handles.pop(canonical_home(home), None)
if handle is not None:
_release_file_lock(handle)
handle.close()
def start_writer(self, thread: threading.Thread) -> None:
"""Register before starting so partial startup cannot forget a live writer."""
with self._mutex:
self._writers.append(thread)
thread.start()
def close(self) -> None:
with self._mutex:
# A timed-out daemon can still write during finally/atexit. Keep the
# handles alive; process exit releases them atomically in the OS.
if any(thread.is_alive() for thread in self._writers):
return
self._writers.clear()
for home in reversed(self.homes):
self.release(home)
process_ownership = ProfileOwnership()
_maintenance = threading.local()
def maintenance_holder(home: Path, purpose: str) -> dict | None:
"""The live holder record of *home*'s ``gateway.lock`` when it is a ``purpose`` maintenance
reservation (see :meth:`ProfileOwnership.reserve`), else None. Advisory only: it names the
holder for a refusal or a short wait, it never grants the lock."""
from gateway.status import _live_pid_from_record, _read_gateway_lock_record
record = _read_gateway_lock_record(canonical_home(home) / 'gateway.lock')
if not record or record.get('purpose') != purpose or _live_pid_from_record(record) is None:
return None
return record
def lock_record_unpublished(home: Path) -> bool:
"""True when *home*'s ``gateway.lock`` record names no live holder yet: empty, torn, or a dead
previous holder's. :meth:`ProfileOwnership.reserve` takes the flock BEFORE it writes the record,
so a contender that loses the flock in that window cannot tell maintenance from a gateway; it
must retry (bounded) instead of reporting a runtime owner. A record naming the CALLER is the
caller's own rolled-back attempt (an all-or-nothing reserve writes each home's record before a
later home refuses), so whoever holds the flock now has not published either. Advisory only,
never grants the lock."""
from gateway.status import _live_pid_from_record, _read_gateway_lock_record
record = _read_gateway_lock_record(canonical_home(home) / 'gateway.lock')
return not record or _live_pid_from_record(record) in (None, os.getpid())
# A client's discovery probe (gateway.status._probe_lock_file) holds a gateway.lock flock for an
# instant; the next attempt after this pause normally finds it gone.
PROBE_RETRY_BACKOFF_S = 0.015
TRANSIENT_HOLDER_POLL_S = 0.1
def _conflicting_homes(homes):
"""The homes whose gateway.lock another holder has right now: each probed alone with its own open
file description and released at once (the all-or-nothing reserve does not say which failed).
A home freed meanwhile is simply not listed; the next reserve attempt takes it."""
conflicting = []
for home in homes:
probe = ProfileOwnership()
try:
probe.reserve([home])
except OwnershipConflict:
conflicting.append(home)
finally:
probe.close()
return conflicting
def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float,
cancel: threading.Event | None = None) -> None:
"""``ownership.reserve(homes)``, retried until *wait_s* passes while every contender is
transient: a client's lock probe (gone by the re-check), a browse reconcile, or a holder that has
not published its record yet (flock taken, record unwritten). A positively identified live owner
on any home refuses at once. Every grant is a real ``reserve``: the flock is never bypassed.
Shared by the gateway boot and the running multiplexer's hot-serve of a new profile.
*cancel* set (a worker thread's caller gave up) ends the wait at once and releases a grant
that landed meanwhile, raising :class:`ReservationCancelled`: the caller may already have run
its ownership cleanup, so nobody would ever release a home taken after it."""
from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
cancel = threading.Event() if cancel is None else cancel
deadline = time.monotonic() + wait_s
logged = False
while not cancel.is_set():
before = set(ownership.homes)
try:
ownership.reserve(homes)
except OwnershipConflict:
# Only the homes that actually conflict decide: a real owner on any of them refuses at
# once, even while another home is mid-browse (multiplexer boot, all-or-nothing).
conflicting = _conflicting_homes(homes)
holders = [h for h in conflicting
if maintenance_holder(h, BROWSE_RECONCILE_PURPOSE) or lock_record_unpublished(h)]
if len(holders) < len(conflicting) or time.monotonic() >= deadline:
raise
if not conflicting:
cancel.wait(PROBE_RETRY_BACKOFF_S) # the holder already let go: a lock probe, never an owner
continue
if not logged:
logger.info("Profile %s is held by a short session-store schema catch-up; waiting up to %.0fs",
holders[0], wait_s)
logged = True
cancel.wait(TRANSIENT_HOLDER_POLL_S)
continue
if not cancel.is_set():
return
for home in {canonical_home(p) for p in homes} - before:
ownership.release(home)
raise ReservationCancelled(f'Reservation of {", ".join(map(str, homes))} cancelled')
def live_runtime_owner(home: Path) -> int | None:
"""PID of a live runtime owner of *home* (a gateway, or a reservation that is not marked as
maintenance), else None. A purpose-marked maintenance holder is never an owner."""
from gateway.status import _live_pid_from_record, _read_gateway_lock_record, get_running_pid
home = canonical_home(home)
record = _read_gateway_lock_record(home / 'gateway.lock')
live = _live_pid_from_record(record) if record else None
if live is not None and record.get('purpose'):
return None # the flock holder is maintenance, so no gateway can hold it too
return get_running_pid(home / 'gateway.pid', cleanup_stale=False) or live
@contextmanager
def exclusive_maintenance(homes, *, purpose: str | None = None):
"""Reserve the authority's exact lock before touching managed state.
This is not an owner-presence check: the reservation remains held through
publication, excluding startup even before its PID/control socket exists.
Only nested synchronous maintenance in this thread may reuse a reservation;
being the runtime owner (even in this process) never grants maintenance.
Lock inodes must not be restored, removed, or replaced by callers.
"""
state = getattr(_maintenance, 'state', None)
outermost = state is None or state[0] != os.getpid()
owner = state[1] if state is not None and not outermost else ProfileOwnership()
previous = set(owner.homes)
try:
owner.reserve(homes, purpose=purpose)
if outermost:
_maintenance.state = (os.getpid(), owner)
yield
except OwnershipConflict as exc:
raise OwnershipConflict(
f'Exclusive maintenance refused: {exc}. Drain and stop the gateway, then retry.'
) from exc
finally:
for home in reversed(owner.homes):
if home not in previous:
owner.release(home)
if outermost:
_maintenance.state = None