The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(sessions): an unusable gateway.lock on a lagging store is a named 503
Tekniumcfa84a960dfc
diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.pyindex 28c91e24270..8cea066f849 100644--- a/hermes_state_reconcile.py+++ b/hermes_state_reconcile.py@@ -253,20 +253,6 @@ def _integrity_problem(path: Path, budget: _Budget):return '; '.join(problems[:3]) or None-def _wait_for_sibling_reconcile(path: Path, home: Path):- """A sibling browse (another thread or process) holds the lock for its own reconcile: wait at- at most the sibling's own execution budget. None once the store is current, else the remaining drift."""- import time- from gateway.runtime_ownership import live_runtime_owner- deadline = time.monotonic() + BROWSE_RECONCILE_BUDGET_S + 0.5- while True:- if (reason := schema_drift_reason(path)) is None:- return None- if time.monotonic() >= deadline or live_runtime_owner(home) is not None:- return reason- time.sleep(_BROWSE_SIBLING_POLL_S)--def _store_signature(path: Path):"""(size, mtime_ns) of the main file and its WAL: any committed write changes one of them."""out = []@@ -292,6 +278,54 @@ def _path_scan_lock(path: Path) -> threading.Lock:return _scan_locks.setdefault(key, threading.Lock())+class _SiblingStillBusy(RuntimeError):+ """Another browse/maintenance holder kept gateway.lock past its own budget; carries the drift."""+++def _reserving(home: Path):+ """``exclusive_maintenance`` for a browse. A lock this user cannot open at all (a root-run+ gateway's, #42685) is not drift and not a crash: refuse by name with its recovery text."""+ from gateway.runtime_ownership import exclusive_maintenance+ with contextlib.ExitStack() as stack:+ try:+ stack.enter_context(exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE))+ except OSError as exc:+ raise DeclarativeDriftError(+ f'Session store is behind this Hermes version, but its gateway.lock cannot be used '+ f'({exc}); nothing was changed.') from exc+ yield+++def _reserve_for_ddl(home: Path, path: Path, owned):+ """Hold gateway.lock for the DDL. A sibling holder (another browse's flock probe or DDL, never a+ gateway) is retried, bounded by its own budget, rather than waited on passively: a probe holds+ the lock for no work and a sibling's DDL finishes inside BROWSE_RECONCILE_BUDGET_S. Yields True+ when the store is already current (nothing to do). Raises ``owned(drift)`` when a gateway turns+ out to own it, and :class:`_SiblingStillBusy` past the bound."""+ import time+ from gateway.runtime_ownership import OwnershipConflict, live_runtime_owner+ deadline = time.monotonic() + BROWSE_RECONCILE_BUDGET_S + 0.5+ with contextlib.ExitStack() as stack:+ while True:+ try:+ stack.enter_context(_reserving(home))+ break+ except OwnershipConflict as exc:+ pending = schema_drift_reason(path)+ if pending is None:+ break # a sibling caught it up; the re-check below yields True without the lock+ if live_runtime_owner(home) is not None:+ raise owned(pending) from exc+ if time.monotonic() >= deadline:+ raise _SiblingStillBusy(pending) from exc+ time.sleep(_BROWSE_SIBLING_POLL_S)+ # Re-check (under the lock when held): a gateway that started and released between probes,+ # or a sibling browse, may already have caught the store up.+ yield schema_drift_reason(path) is None++def reconcile_drift_for_browse(path: Path) -> bool:"""Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.@@ -299,7 +333,7 @@ def reconcile_drift_for_browse(path: Path) -> bool:with a user-facing detail naming the drift and the remedy, and :class:`CorruptStoreError`when the lagging store is also damaged. Never touches a store missing its history tables:recreating them empty would hide lost sessions."""- from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance, live_runtime_owner+ from gateway.runtime_ownership import OwnershipConflict, live_runtime_ownerfrom hermes_state_health import STORAGE_CORRUPT, mark_storage_corrupt, storage_corrupt_reason, storage_statereason = schema_drift_reason(path)@@ -332,7 +366,7 @@ def reconcile_drift_for_browse(path: Path) -> bool:# A gateway already owns it: never scan or write, refuse by name at once. Probe the flock itself# (a record left by a released holder still names a live PID), holding it for no work at all.try:- with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):+ with _reserving(home):passexcept OwnershipConflict as exc:if live_runtime_owner(home) is not None:@@ -359,12 +393,10 @@ def reconcile_drift_for_browse(path: Path) -> bool:mark_storage_corrupt(path, problem)raise CorruptStoreError(problem)try:- with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):- budget = _Budget(BROWSE_RECONCILE_BUDGET_S)- # Re-check under the lock: a gateway that started and released between probes- # (or a sibling browse) may already have caught the store up.- if schema_drift_reason(path) is None:+ with _reserve_for_ddl(home, path, owned) as current:+ if current:return False+ budget = _Budget(BROWSE_RECONCILE_BUDGET_S)if _store_signature(path) != signature:# Written since the scan (a gateway that came and went): the scan no longer# vouches for these bytes. Refuse; the next browse scans the new state.@@ -380,17 +412,10 @@ def reconcile_drift_for_browse(path: Path) -> bool:'profile.') from excexcept BrowseBudgetExceeded as exc:raise too_slow(f'caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s') from exc- except OwnershipConflict as exc:- if live_runtime_owner(home) is not None:- raise owned(reason) from exc- # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).- if (pending := _wait_for_sibling_reconcile(path, home)) is None:- return False- if live_runtime_owner(home) is not None:- raise owned(pending) from exc+ except _SiblingStillBusy as exc:raise DeclarativeDriftError(f"Session store for profile '{label}' is being brought up to date by another Hermes "- f'process ({pending}). Retry in a moment; the list was not cleared.') from exc+ f'process ({exc}). Retry in a moment; the list was not cleared.') from excif after := schema_drift_reason(path):raise DeclarativeDriftError(f"Session store for profile '{label}' is behind this Hermes version and could not be "diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.pyindex bfd5cde1bf2..af93176a6e2 100644--- a/tests/hermes_cli/test_browse_schema_drift.py+++ b/tests/hermes_cli/test_browse_schema_drift.py@@ -798,3 +798,30 @@ def test_slow_integrity_scan_is_bounded_and_not_repeated_on_every_poll(tmp_path,assert "nothing was changed" in caught.value.detail and "Retry" in caught.value.detailassert scans == [1]assert _file_state(path) == before++[email protected]("linux", "macos")+def test_unopenable_lock_on_lagging_store_is_a_named_503(tmp_path):+ """pastels m1: a gateway.lock this user cannot open (left by a root-run gateway, #42685) raised+ PermissionError(foreign_stale_lock) straight out of the browse open: an unhandled 500 on+ /api/sessions and /messages?profile= where main answered 503. It must be a named 503 carrying+ the recovery text, with nothing written."""+ import os++ if os.geteuid() == 0:+ pytest.skip("root opens any file")+ home = _profile_home(tmp_path)+ path = _lagging_store_with_history(home, rows=10)+ lock = home / "gateway.lock"+ lock.write_text("")+ lock.chmod(0)+ before = _file_state(path)+ try:+ with pytest.raises(HTTPException) as caught:+ _open_session_db_at_path(path, read_only=True)+ finally:+ lock.chmod(0o600)+ assert caught.value.status_code == 503+ detail = str(caught.value.detail)+ assert "foreign_stale_lock" in detail and "nothing was changed" in detail+ assert _file_state(path) == before