The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(sessions): sibling browses wait for each other instead of naming a gateway
Teknium66906f5a5136
diff --git a/gateway/runtime_ownership.py b/gateway/runtime_ownership.pyindex d3bdd821b97..33b678d4cd1 100644--- a/gateway/runtime_ownership.py+++ b/gateway/runtime_ownership.py@@ -148,7 +148,9 @@ class ProfileOwnership:with self._mutex:return tuple(self._handles)- def reserve(self, homes) -> None:+ def reserve(self, homes, *, purpose: str | None = None) -> None:+ """``purpose`` marks a maintenance reservation in the lock record, so a contender can tell+ a short maintenance holder from a gateway runtime owner (:func:`live_runtime_owner`)."""from gateway.status import _try_acquire_file_lock, _build_pid_recordwith self._mutex:added = []@@ -164,6 +166,8 @@ class ProfileOwnership:if not _try_acquire_file_lock(handle) or not _still_names(path, handle.fileno()):raise OwnershipConflict(f'Gateway runtime already owns profile {home}')record = {**_build_pid_record(), 'hermes_home': str(home)}+ if purpose:+ record['purpose'] = purposehandle.seek(0)handle.truncate()json.dump(record, handle)@@ -212,8 +216,20 @@ process_ownership = ProfileOwnership()_maintenance = threading.local()+def live_runtime_owner(home: Path) -> int | None:+ """PID of a live runtime owner of *home* (a gateway, or a reservation that is not marked as+ maintenance), else None. A purpose-marked maintenance holder is never an owner."""+ from gateway.status import _live_pid_from_record, _read_gateway_lock_record, get_running_pid+ home = canonical_home(home)+ record = _read_gateway_lock_record(home / 'gateway.lock')+ live = _live_pid_from_record(record) if record else None+ if live is not None and record.get('purpose'):+ return None # the flock holder is maintenance, so no gateway can hold it too+ return get_running_pid(home / 'gateway.pid', cleanup_stale=False) or live++@contextmanager-def exclusive_maintenance(homes):+def exclusive_maintenance(homes, *, purpose: str | None = None):"""Reserve the authority's exact lock before touching managed state.This is not an owner-presence check: the reservation remains held through@@ -227,7 +243,7 @@ def exclusive_maintenance(homes):owner = state[1] if state is not None and not outermost else ProfileOwnership()previous = set(owner.homes)try:- owner.reserve(homes)+ owner.reserve(homes, purpose=purpose)if outermost:_maintenance.state = (os.getpid(), owner)yielddiff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.pyindex ac94f20e68b..5f9d0f0b803 100644--- a/hermes_state_reconcile.py+++ b/hermes_state_reconcile.py@@ -157,6 +157,10 @@ def _apply_additive_schema(path: Path) -> None:raise+BROWSE_RECONCILE_PURPOSE = 'browse_schema_reconcile'+_BROWSE_SIBLING_POLL_S = 0.1++class CorruptStoreError(sqlite3.DatabaseError):"""The lagging store is also damaged: browsing must not write DDL into it. Worded like sqlite'sown malformed-image error so every caller maps it to the corrupt-store status (``hermes doctor``)."""@@ -180,6 +184,20 @@ def _quick_check_problem(path: Path):return '; '.join(problems[:3]) or None+def _wait_for_sibling_reconcile(path: Path, home: Path):+ """A sibling browse (another thread or process) holds the lock for its own reconcile: wait at+ most the busy budget for it. None once the store is current, else the remaining drift."""+ import time+ from gateway.runtime_ownership import live_runtime_owner+ deadline = time.monotonic() + _BROWSE_RECONCILE_BUSY_S+ while True:+ if (reason := schema_drift_reason(path)) is None:+ return None+ if time.monotonic() >= deadline or live_runtime_owner(home) is not None:+ return reason+ time.sleep(_BROWSE_SIBLING_POLL_S)++def reconcile_drift_for_browse(path: Path) -> bool:"""Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.@@ -187,7 +205,7 @@ def reconcile_drift_for_browse(path: Path) -> bool:with a user-facing detail naming the drift and the remedy, and :class:`CorruptStoreError`when the lagging store is also damaged. Never touches a store missing its history tables:recreating them empty would hide lost sessions."""- from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance+ from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance, live_runtime_ownerreason = schema_drift_reason(path)if reason is None:@@ -200,9 +218,16 @@ def reconcile_drift_for_browse(path: Path) -> bool:raise DeclarativeDriftError(f'Session store is missing its history tables ({reason}); this is not upgrade drift. ''Run `hermes sessions recover`.') from exc- label, remedy = _owner_remedy(path.parent)+ home = path.parent+ label, remedy = _owner_remedy(home)++ def owned(drift):+ return DeclarativeDriftError(+ f"Session store for profile '{label}' is behind this Hermes version ({drift}), and a "+ f'running gateway owns it, so browsing will not write to it. Restart that gateway '+ f'(`{remedy}`) to bring the store up to date.', owned=True)try:- with exclusive_maintenance([path.parent]):+ with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):# Re-check under the lock: a gateway that started and released between probes# (or a sibling browse) may already have caught the store up.if schema_drift_reason(path) is None:@@ -213,10 +238,16 @@ def reconcile_drift_for_browse(path: Path) -> bool:raise CorruptStoreError(problem)_apply_additive_schema(path)except OwnershipConflict as exc:+ if live_runtime_owner(home) is not None:+ raise owned(reason) from exc+ # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).+ if (pending := _wait_for_sibling_reconcile(path, home)) is None:+ return False+ if live_runtime_owner(home) is not None:+ raise owned(pending) from excraise DeclarativeDriftError(- f"Session store for profile '{label}' is behind this Hermes version ({reason}), and a "- f'running gateway owns it, so browsing will not write to it. Restart that gateway '- f'(`{remedy}`) to bring the store up to date.', owned=True) from exc+ f"Session store for profile '{label}' is being brought up to date by another Hermes "+ f'process ({pending}). Retry in a moment; the list was not cleared.') from excif after := schema_drift_reason(path):raise DeclarativeDriftError(f"Session store for profile '{label}' is behind this Hermes version and could not be "diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.pyindex 29b833477ea..f47690f0e05 100644--- a/tests/hermes_cli/test_browse_schema_drift.py+++ b/tests/hermes_cli/test_browse_schema_drift.py@@ -307,3 +307,106 @@ def test_corrupt_non_root_page_plus_drift_is_not_written(tmp_path):assert caught.value.detail["error"] == "state_db_corrupt"assert _file_state(path) == beforeassert "last_read_at" not in _session_columns(path)+++def test_concurrent_browses_without_a_gateway_all_succeed(tmp_path):+ """P2-1: sibling browses of one lagging, unowned store contend for gateway.lock with each+ other. None may be told a running gateway owns the store; each waits for the sibling."""+ import threading++ path = _lagging_store_with_history(_profile_home(tmp_path), rows=50)+ barrier = threading.Barrier(8)+ results = []++ def browse():+ barrier.wait()+ try:+ _open_session_db_at_path(path, read_only=True).close()+ results.append("200")+ except HTTPException as exc:+ results.append(f"{exc.status_code} {exc.detail}")++ threads = [threading.Thread(target=browse) for _ in range(8)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ assert results == ["200"] * 8, results+ assert "last_read_at" in _session_columns(path)+++_PROC_BROWSE = r'''+import sys, time+from pathlib import Path+from fastapi import HTTPException+from hermes_cli.web_server_sessions import _open_session_db_at_path+go = float(sys.argv[2])+while time.time() < go:+ time.sleep(0.0005)+try:+ _open_session_db_at_path(Path(sys.argv[1]), read_only=True).close()+ print("RESULT 200")+except HTTPException as exc:+ print("RESULT", exc.status_code, exc.detail)+'''++[email protected]("linux", "macos")+def test_concurrent_browses_from_processes_all_succeed(tmp_path):+ import os+ import subprocess+ import sys+ import time++ path = _lagging_store_with_history(_profile_home(tmp_path), rows=50)+ go = time.time() + 2.0+ procs = [subprocess.Popen([sys.executable, "-c", _PROC_BROWSE, str(path), str(go)], env=os.environ,+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) for _ in range(5)]+ out = [p.communicate(timeout=60)[0] for p in procs]+ results = [line for o in out for line in o.splitlines() if line.startswith("RESULT")]+ assert results == ["RESULT 200"] * 5, out+++_PURPOSE_HOLDER = r'''+import sys, time+from pathlib import Path+from gateway.runtime_ownership import ProfileOwnership+purpose = sys.argv[3] or None+owner = ProfileOwnership()+owner.reserve([Path(sys.argv[1])], purpose=purpose)+print("held", flush=True)+time.sleep(float(sys.argv[2]))+'''+++def _hold(home, seconds, purpose=""):+ import os+ import subprocess+ import sys+ proc = subprocess.Popen([sys.executable, "-c", _PURPOSE_HOLDER, str(home), str(seconds), purpose],+ env=os.environ, stdout=subprocess.PIPE, text=True)+ assert proc.stdout.readline().strip() == "held"+ return proc++[email protected]("linux", "macos")+def test_stuck_maintenance_holder_gets_busy_retry_not_a_gateway_remedy(tmp_path):+ """P2-1: when the holder is not a gateway and the store is still behind after the busy budget,+ the refusal says busy/retry and never tells the user to restart a gateway that is not running."""+ from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE++ home = _profile_home(tmp_path)+ path = _lagging_store_with_history(home, rows=10)+ holder = _hold(home, 30, BROWSE_RECONCILE_PURPOSE)+ try:+ before = _file_state(path)+ with pytest.raises(HTTPException) as caught:+ _open_session_db_at_path(path, read_only=True)+ finally:+ holder.kill()+ holder.wait()+ detail = str(caught.value.detail)+ assert caught.value.status_code == 503+ assert "Retry" in detail+ assert "gateway" not in detail.lower()+ assert _file_state(path) == before