Skip to content

The agent that grows with you

fix(sessions): sibling browses wait for each other instead of naming a gateway

Teknium66906f5a5136
diff --git a/gateway/runtime_ownership.py b/gateway/runtime_ownership.py
index d3bdd821b97..33b678d4cd1 100644
--- a/gateway/runtime_ownership.py
+++ b/gateway/runtime_ownership.py
@@ -148,7 +148,9 @@ class ProfileOwnership:
with self._mutex:
return tuple(self._handles)
- def reserve(self, homes) -> None:
+ def reserve(self, homes, *, purpose: str | None = None) -> None:
+ """``purpose`` marks a maintenance reservation in the lock record, so a contender can tell
+ a short maintenance holder from a gateway runtime owner (:func:`live_runtime_owner`)."""
from gateway.status import _try_acquire_file_lock, _build_pid_record
with self._mutex:
added = []
@@ -164,6 +166,8 @@ class ProfileOwnership:
if not _try_acquire_file_lock(handle) or not _still_names(path, handle.fileno()):
raise OwnershipConflict(f'Gateway runtime already owns profile {home}')
record = {**_build_pid_record(), 'hermes_home': str(home)}
+ if purpose:
+ record['purpose'] = purpose
handle.seek(0)
handle.truncate()
json.dump(record, handle)
@@ -212,8 +216,20 @@ process_ownership = ProfileOwnership()
_maintenance = threading.local()
+def live_runtime_owner(home: Path) -> int | None:
+ """PID of a live runtime owner of *home* (a gateway, or a reservation that is not marked as
+ maintenance), else None. A purpose-marked maintenance holder is never an owner."""
+ from gateway.status import _live_pid_from_record, _read_gateway_lock_record, get_running_pid
+ home = canonical_home(home)
+ record = _read_gateway_lock_record(home / 'gateway.lock')
+ live = _live_pid_from_record(record) if record else None
+ if live is not None and record.get('purpose'):
+ return None # the flock holder is maintenance, so no gateway can hold it too
+ return get_running_pid(home / 'gateway.pid', cleanup_stale=False) or live
+
+
@contextmanager
-def exclusive_maintenance(homes):
+def exclusive_maintenance(homes, *, purpose: str | None = None):
"""Reserve the authority's exact lock before touching managed state.
This is not an owner-presence check: the reservation remains held through
@@ -227,7 +243,7 @@ def exclusive_maintenance(homes):
owner = state[1] if state is not None and not outermost else ProfileOwnership()
previous = set(owner.homes)
try:
- owner.reserve(homes)
+ owner.reserve(homes, purpose=purpose)
if outermost:
_maintenance.state = (os.getpid(), owner)
yield
diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.py
index ac94f20e68b..5f9d0f0b803 100644
--- a/hermes_state_reconcile.py
+++ b/hermes_state_reconcile.py
@@ -157,6 +157,10 @@ def _apply_additive_schema(path: Path) -> None:
raise
+BROWSE_RECONCILE_PURPOSE = 'browse_schema_reconcile'
+_BROWSE_SIBLING_POLL_S = 0.1
+
+
class CorruptStoreError(sqlite3.DatabaseError):
"""The lagging store is also damaged: browsing must not write DDL into it. Worded like sqlite's
own malformed-image error so every caller maps it to the corrupt-store status (``hermes doctor``)."""
@@ -180,6 +184,20 @@ def _quick_check_problem(path: Path):
return '; '.join(problems[:3]) or None
+def _wait_for_sibling_reconcile(path: Path, home: Path):
+ """A sibling browse (another thread or process) holds the lock for its own reconcile: wait at
+ most the busy budget for it. None once the store is current, else the remaining drift."""
+ import time
+ from gateway.runtime_ownership import live_runtime_owner
+ deadline = time.monotonic() + _BROWSE_RECONCILE_BUSY_S
+ while True:
+ if (reason := schema_drift_reason(path)) is None:
+ return None
+ if time.monotonic() >= deadline or live_runtime_owner(home) is not None:
+ return reason
+ time.sleep(_BROWSE_SIBLING_POLL_S)
+
+
def reconcile_drift_for_browse(path: Path) -> bool:
"""Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.
@@ -187,7 +205,7 @@ def reconcile_drift_for_browse(path: Path) -> bool:
with a user-facing detail naming the drift and the remedy, and :class:`CorruptStoreError`
when the lagging store is also damaged. Never touches a store missing its history tables:
recreating them empty would hide lost sessions."""
- from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
+ from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance, live_runtime_owner
reason = schema_drift_reason(path)
if reason is None:
@@ -200,9 +218,16 @@ def reconcile_drift_for_browse(path: Path) -> bool:
raise DeclarativeDriftError(
f'Session store is missing its history tables ({reason}); this is not upgrade drift. '
'Run `hermes sessions recover`.') from exc
- label, remedy = _owner_remedy(path.parent)
+ home = path.parent
+ label, remedy = _owner_remedy(home)
+
+ def owned(drift):
+ return DeclarativeDriftError(
+ f"Session store for profile '{label}' is behind this Hermes version ({drift}), and a "
+ f'running gateway owns it, so browsing will not write to it. Restart that gateway '
+ f'(`{remedy}`) to bring the store up to date.', owned=True)
try:
- with exclusive_maintenance([path.parent]):
+ with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):
# Re-check under the lock: a gateway that started and released between probes
# (or a sibling browse) may already have caught the store up.
if schema_drift_reason(path) is None:
@@ -213,10 +238,16 @@ def reconcile_drift_for_browse(path: Path) -> bool:
raise CorruptStoreError(problem)
_apply_additive_schema(path)
except OwnershipConflict as exc:
+ if live_runtime_owner(home) is not None:
+ raise owned(reason) from exc
+ # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).
+ if (pending := _wait_for_sibling_reconcile(path, home)) is None:
+ return False
+ if live_runtime_owner(home) is not None:
+ raise owned(pending) from exc
raise DeclarativeDriftError(
- f"Session store for profile '{label}' is behind this Hermes version ({reason}), and a "
- f'running gateway owns it, so browsing will not write to it. Restart that gateway '
- f'(`{remedy}`) to bring the store up to date.', owned=True) from exc
+ f"Session store for profile '{label}' is being brought up to date by another Hermes "
+ f'process ({pending}). Retry in a moment; the list was not cleared.') from exc
if after := schema_drift_reason(path):
raise DeclarativeDriftError(
f"Session store for profile '{label}' is behind this Hermes version and could not be "
diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.py
index 29b833477ea..f47690f0e05 100644
--- a/tests/hermes_cli/test_browse_schema_drift.py
+++ b/tests/hermes_cli/test_browse_schema_drift.py
@@ -307,3 +307,106 @@ def test_corrupt_non_root_page_plus_drift_is_not_written(tmp_path):
assert caught.value.detail["error"] == "state_db_corrupt"
assert _file_state(path) == before
assert "last_read_at" not in _session_columns(path)
+
+
+def test_concurrent_browses_without_a_gateway_all_succeed(tmp_path):
+ """P2-1: sibling browses of one lagging, unowned store contend for gateway.lock with each
+ other. None may be told a running gateway owns the store; each waits for the sibling."""
+ import threading
+
+ path = _lagging_store_with_history(_profile_home(tmp_path), rows=50)
+ barrier = threading.Barrier(8)
+ results = []
+
+ def browse():
+ barrier.wait()
+ try:
+ _open_session_db_at_path(path, read_only=True).close()
+ results.append("200")
+ except HTTPException as exc:
+ results.append(f"{exc.status_code} {exc.detail}")
+
+ threads = [threading.Thread(target=browse) for _ in range(8)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert results == ["200"] * 8, results
+ assert "last_read_at" in _session_columns(path)
+
+
+_PROC_BROWSE = r'''
+import sys, time
+from pathlib import Path
+from fastapi import HTTPException
+from hermes_cli.web_server_sessions import _open_session_db_at_path
+go = float(sys.argv[2])
+while time.time() < go:
+ time.sleep(0.0005)
+try:
+ _open_session_db_at_path(Path(sys.argv[1]), read_only=True).close()
+ print("RESULT 200")
+except HTTPException as exc:
+ print("RESULT", exc.status_code, exc.detail)
+'''
+
+
[email protected]("linux", "macos")
+def test_concurrent_browses_from_processes_all_succeed(tmp_path):
+ import os
+ import subprocess
+ import sys
+ import time
+
+ path = _lagging_store_with_history(_profile_home(tmp_path), rows=50)
+ go = time.time() + 2.0
+ procs = [subprocess.Popen([sys.executable, "-c", _PROC_BROWSE, str(path), str(go)], env=os.environ,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) for _ in range(5)]
+ out = [p.communicate(timeout=60)[0] for p in procs]
+ results = [line for o in out for line in o.splitlines() if line.startswith("RESULT")]
+ assert results == ["RESULT 200"] * 5, out
+
+
+_PURPOSE_HOLDER = r'''
+import sys, time
+from pathlib import Path
+from gateway.runtime_ownership import ProfileOwnership
+purpose = sys.argv[3] or None
+owner = ProfileOwnership()
+owner.reserve([Path(sys.argv[1])], purpose=purpose)
+print("held", flush=True)
+time.sleep(float(sys.argv[2]))
+'''
+
+
+def _hold(home, seconds, purpose=""):
+ import os
+ import subprocess
+ import sys
+ proc = subprocess.Popen([sys.executable, "-c", _PURPOSE_HOLDER, str(home), str(seconds), purpose],
+ env=os.environ, stdout=subprocess.PIPE, text=True)
+ assert proc.stdout.readline().strip() == "held"
+ return proc
+
+
[email protected]("linux", "macos")
+def test_stuck_maintenance_holder_gets_busy_retry_not_a_gateway_remedy(tmp_path):
+ """P2-1: when the holder is not a gateway and the store is still behind after the busy budget,
+ the refusal says busy/retry and never tells the user to restart a gateway that is not running."""
+ from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
+
+ home = _profile_home(tmp_path)
+ path = _lagging_store_with_history(home, rows=10)
+ holder = _hold(home, 30, BROWSE_RECONCILE_PURPOSE)
+ try:
+ before = _file_state(path)
+ with pytest.raises(HTTPException) as caught:
+ _open_session_db_at_path(path, read_only=True)
+ finally:
+ holder.kill()
+ holder.wait()
+ detail = str(caught.value.detail)
+ assert caught.value.status_code == 503
+ assert "Retry" in detail
+ assert "gateway" not in detail.lower()
+ assert _file_state(path) == before