The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(sessions): browse integrity scan runs before gateway.lock, never under it
Teknium7ddf40fee8ac
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.pyindex 9edbe3b75b3..53413907080 100644--- a/gateway/run_bootstrap.py+++ b/gateway/run_bootstrap.py@@ -545,10 +545,11 @@ def _start_gateway_resolve_terminal_cwd() -> None:os.environ["TERMINAL_CWD"] = resolved_cwd-# A browse that catches a lagging state.db up holds the profile's gateway.lock for at most its-# execution budget (hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S: integrity check + DDL, cut off-# by a sqlite progress handler). A gateway booting in that window waits it out, with margin for-# lock publication and the drift probes, instead of exiting as if another gateway owned the profile.+# A browse that catches a lagging state.db up holds the profile's gateway.lock only for the drift+# re-check and the additive DDL, cut off at hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S by a+# sqlite progress handler. Its integrity scan (which scales with file size) runs BEFORE the lock, so+# a large store never extends this window. A gateway booting in it waits it out, with margin for lock+# publication and the drift probes, instead of exiting as if another gateway owned the profile.def _browse_reconcile_reserve_wait_s() -> float:from hermes_state_reconcile import BROWSE_RECONCILE_BUDGET_Sreturn BROWSE_RECONCILE_BUDGET_S + 2.0diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.pyindex 1f91b82bfb8..28c91e24270 100644--- a/hermes_state_reconcile.py+++ b/hermes_state_reconcile.py@@ -8,6 +8,7 @@ surgery (:func:`repair_declarative_schema`). Neither runs while a gateway owns timport contextlibimport loggingimport sqlite3+import threadingfrom contextlib import closingfrom pathlib import Path@@ -104,12 +105,15 @@ def _reconcile_locked(path, backup, report):# profile's gateway runtime lock. When a gateway owns the store it is the only writer: refuse# by name with the remedy instead of becoming a second one.#-# The lock is held for at most BROWSE_RECONCILE_BUDGET_S: one monotonic deadline, starting once the-# lock is held, covers the integrity check and the DDL transaction. A sqlite progress handler-# interrupts either one past the deadline (the DDL rolls back) and browse answers a retryable 503.-# Busy waiting does not run the progress handler, so each connection's busy timeout is capped at-# the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).+# The full integrity scan is read-only and scales with file size, so it runs BEFORE the lock, on its+# own budget (BROWSE_INTEGRITY_BUDGET_S); a gateway booting meanwhile simply takes the lock. The+# scan is tied to the file's (size, mtime) signature: if the store changed by the time the lock is+# held, the scan no longer vouches for it and nothing is written. Under the lock only the cheap drift+# re-check and the DDL run, cut off by BROWSE_RECONCILE_BUDGET_S (a sqlite progress handler; the DDL+# rolls back). Busy waiting does not run the progress handler, so each connection's busy timeout is+# capped at the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).BROWSE_RECONCILE_BUDGET_S = 3.0+BROWSE_INTEGRITY_BUDGET_S = 20.0_BUDGET_PROGRESS_OPS = 1000@@ -263,6 +267,31 @@ def _wait_for_sibling_reconcile(path: Path, home: Path):time.sleep(_BROWSE_SIBLING_POLL_S)+def _store_signature(path: Path):+ """(size, mtime_ns) of the main file and its WAL: any committed write changes one of them."""+ out = []+ for p in (path, path.with_name(path.name + '-wal')):+ try:+ st = p.stat()+ out.append((st.st_size, st.st_mtime_ns))+ except FileNotFoundError:+ out.append(None)+ return tuple(out)+++_scan_guard = threading.Lock()+_scan_locks: dict[str, threading.Lock] = {}+# Signatures whose integrity scan could not finish inside BROWSE_INTEGRITY_BUDGET_S. A sidebar polls+# every few seconds; rescanning a multi-GB file on each poll only burns I/O for the same answer.+_scan_too_slow: dict[str, tuple] = {}+++def _path_scan_lock(path: Path) -> threading.Lock:+ key = str(path.resolve())+ with _scan_guard:+ return _scan_locks.setdefault(key, threading.Lock())++def reconcile_drift_for_browse(path: Path) -> bool:"""Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.@@ -271,6 +300,7 @@ def reconcile_drift_for_browse(path: Path) -> bool:when the lagging store is also damaged. Never touches a store missing its history tables:recreating them empty would hide lost sessions."""from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance, live_runtime_owner+ from hermes_state_health import STORAGE_CORRUPT, mark_storage_corrupt, storage_corrupt_reason, storage_statereason = schema_drift_reason(path)if reason is None:@@ -291,41 +321,76 @@ def reconcile_drift_for_browse(path: Path) -> bool:f"Session store for profile '{label}' is behind this Hermes version ({drift}), and a "f'running gateway owns it, so browsing will not write to it. Restart that gateway 'f'(`{remedy}`) to bring the store up to date.', owned=True)++ def too_slow(what):+ return DeclarativeDriftError(+ f"Session store for profile '{label}' is behind this Hermes version ({reason}) and could "+ f'not be {what} while browsing; nothing was changed and the list was not cleared. Retry, '+ f"or start this profile's gateway (`{remedy}`) or run `hermes sessions repair` to bring "+ 'it up to date.')++ # A gateway already owns it: never scan or write, refuse by name at once. Probe the flock itself+ # (a record left by a released holder still names a live PID), holding it for no work at all.try:with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):- budget = _Budget(BROWSE_RECONCILE_BUDGET_S)- # Re-check under the lock: a gateway that started and released between probes- # (or a sibling browse) may already have caught the store up.- if schema_drift_reason(path) is None:- return False- # A failing column probe hides damage elsewhere in the file: additive DDL is not- # corruption recovery, so never write into a store that is not verified clean.- if problem := _integrity_problem(path, budget):- raise CorruptStoreError(problem)- try:- _apply_additive_schema(path, budget)- except DeclarativeDriftError as exc:- raise DeclarativeDriftError(- f"Session store for profile '{label}' is behind this Hermes version and could not "- f'be caught up ({exc}); nothing was changed. Run `hermes sessions repair` for that '- 'profile.') from exc- except BrowseBudgetExceeded as exc:- raise DeclarativeDriftError(- f"Session store for profile '{label}' is behind this Hermes version ({reason}) and could "- f'not be verified and caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s while browsing; '- f'nothing was changed and the list was not cleared. Retry, or start this profile\'s '- f'gateway (`{remedy}`) or run `hermes sessions repair` to bring it up to date.') from exc+ passexcept OwnershipConflict as exc:if live_runtime_owner(home) is not None:raise owned(reason) from exc- # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).- if (pending := _wait_for_sibling_reconcile(path, home)) is None:+ # Already latched corrupt in this process: do not rescan a damaged file on every poll.+ if storage_state(path) == STORAGE_CORRUPT:+ raise CorruptStoreError(storage_corrupt_reason(path) or 'latched corrupt')+ key = str(path.resolve())+ with _path_scan_lock(path): # one scan per store per process; concurrent polls wait for it+ if schema_drift_reason(path) is None:return False- if live_runtime_owner(home) is not None:- raise owned(pending) from exc- raise DeclarativeDriftError(- f"Session store for profile '{label}' is being brought up to date by another Hermes "- f'process ({pending}). Retry in a moment; the list was not cleared.') from exc+ signature = _store_signature(path)+ if _scan_too_slow.get(key) == signature:+ raise too_slow(f'verified within {BROWSE_INTEGRITY_BUDGET_S:.0f}s')+ # Outside gateway.lock: read-only, may be slow on a large store, never blocks a gateway boot.+ try:+ problem = _integrity_problem(path, _Budget(BROWSE_INTEGRITY_BUDGET_S))+ except BrowseBudgetExceeded as exc:+ _scan_too_slow[key] = signature+ raise too_slow(f'verified within {BROWSE_INTEGRITY_BUDGET_S:.0f}s') from exc+ if problem:+ # A failing column probe hides damage elsewhere in the file: additive DDL is not+ # corruption recovery, so never write into a store that is not verified clean.+ mark_storage_corrupt(path, problem)+ raise CorruptStoreError(problem)+ try:+ with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):+ budget = _Budget(BROWSE_RECONCILE_BUDGET_S)+ # Re-check under the lock: a gateway that started and released between probes+ # (or a sibling browse) may already have caught the store up.+ if schema_drift_reason(path) is None:+ return False+ if _store_signature(path) != signature:+ # Written since the scan (a gateway that came and went): the scan no longer+ # vouches for these bytes. Refuse; the next browse scans the new state.+ raise DeclarativeDriftError(+ f"Session store for profile '{label}' changed while browsing verified it "+ f'({reason}); nothing was changed. Retry in a moment; the list was not cleared.')+ try:+ _apply_additive_schema(path, budget)+ except DeclarativeDriftError as exc:+ raise DeclarativeDriftError(+ f"Session store for profile '{label}' is behind this Hermes version and could not "+ f'be caught up ({exc}); nothing was changed. Run `hermes sessions repair` for that '+ 'profile.') from exc+ except BrowseBudgetExceeded as exc:+ raise too_slow(f'caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s') from exc+ except OwnershipConflict as exc:+ if live_runtime_owner(home) is not None:+ raise owned(reason) from exc+ # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).+ if (pending := _wait_for_sibling_reconcile(path, home)) is None:+ return False+ if live_runtime_owner(home) is not None:+ raise owned(pending) from exc+ raise DeclarativeDriftError(+ f"Session store for profile '{label}' is being brought up to date by another Hermes "+ f'process ({pending}). Retry in a moment; the list was not cleared.') from excif after := schema_drift_reason(path):raise DeclarativeDriftError(f"Session store for profile '{label}' is behind this Hermes version and could not be "diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.pyindex 96612059f34..bfd5cde1bf2 100644--- a/tests/hermes_cli/test_browse_schema_drift.py+++ b/tests/hermes_cli/test_browse_schema_drift.py@@ -546,6 +546,7 @@ def slowed(path, *a, **kw):conn = original(path, *a, **kw)read_only = "mode=ro" in str(path)if holding_lock() and (phase == "integrity") == read_only:+ print("slowed", flush=True) # this process holds gateway.lock and the slow phase beginsreturn SlowConnection(conn)return conn@@ -561,11 +562,12 @@ except HTTPException as exc:@pytest.mark.platforms("linux", "macos")[email protected]("phase", ["integrity", "ddl"])[email protected]("phase", ["ddl"])def test_slow_browse_reconcile_gives_up_inside_its_budget_and_boot_waits_it_out(tmp_path, phase):- """R2-2: the busy timeout bounds lock contention, not SQL execution. A slow integrity scan or- DDL must be interrupted at the browse budget (rolled back, retryable 503), and gateway boot's- wait must outlast that budget, so a slow browse never turns into a startup refusal."""+ """R2-2: the busy timeout bounds lock contention, not SQL execution. A slow DDL must be+ interrupted at the browse budget (rolled back, retryable 503), and gateway boot's wait must+ outlast that budget, so a slow browse never turns into a startup refusal. (The integrity scan+ no longer runs under the lock: see test_gateway_boot_never_waits_on_a_slow_integrity_scan.)"""import osimport subprocessimport sys@@ -584,10 +586,7 @@ def test_slow_browse_reconcile_gives_up_inside_its_budget_and_boot_waits_it_out(browse = subprocess.Popen([sys.executable, "-c", _SLOW_BROWSE, str(path), phase], env=os.environ,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)assert browse.stdout.readline().strip() == "started"- deadline = time.monotonic() + 20- while not (home / "gateway.lock").exists() or not (home / "gateway.lock").read_text():- assert time.monotonic() < deadline and browse.poll() is None- time.sleep(0.01)+ assert browse.stdout.readline().strip() == "slowed", "the slow phase never ran under the lock"booting = ProfileOwnership()try:bootstrap._reserve_waiting_out_browse_reconcile(booting, [home]) # default wait: must not raise@@ -653,3 +652,149 @@ def test_failed_column_reconcile_rolls_back_every_addition(tmp_path):assert "nothing was changed" in str(caught.value.detail)assert _file_state(path) == beforeassert "last_read_at" not in _session_columns(path)+++# ── Maintainer review round 3 ──+++def test_integrity_scan_runs_before_the_lock_is_taken(tmp_path, monkeypatch):+ """alt-glitch C1 / pastels m3: the full integrity scan is read-only and scales with file size.+ It must run BEFORE gateway.lock is reserved, so a booting gateway never waits on it."""+ import hermes_state_reconcile as reconcile+ from gateway.runtime_ownership import OwnershipConflict, ProfileOwnership++ home = _profile_home(tmp_path)+ path = _lagging_store_with_history(home, rows=10)+ original = reconcile._integrity_problem+ lock_free_during_scan = []++ def spy(p, budget):+ other = ProfileOwnership() # its own open file description: conflicts with any holder+ try:+ other.reserve([home])+ lock_free_during_scan.append(True)+ except OwnershipConflict:+ lock_free_during_scan.append(False)+ finally:+ other.close()+ return original(p, budget)++ monkeypatch.setattr(reconcile, "_integrity_problem", spy)+ _open_session_db_at_path(path, read_only=True).close()+ assert lock_free_during_scan == [True]+ assert "last_read_at" in _session_columns(path)+++_SLOW_SCAN_BROWSE = r'''+import sys, time+from pathlib import Path+import hermes_cli.sqlite_safe_read as ssr+from fastapi import HTTPException+from hermes_cli.web_server_sessions import _open_session_db_at_path++original = ssr.connect_tracked++class SlowScan:+ """A real read-only connection whose integrity scan runs slowly (a cold multi-GB store)."""+ def __init__(self, conn):+ self._conn = conn+ def __getattr__(self, name):+ return getattr(self._conn, name)+ def execute(self, sql, *a):+ if sql.startswith("PRAGMA integrity_check"):+ time.sleep(float(sys.argv[2]))+ return self._conn.execute(sql, *a)+ def close(self):+ return self._conn.close()++def slowed(path, *a, **kw):+ conn = original(path, *a, **kw)+ return SlowScan(conn) if "mode=ro" in str(path) else conn++ssr.connect_tracked = slowed+print("started", flush=True)+try:+ _open_session_db_at_path(Path(sys.argv[1]), read_only=True).close()+ print("RESULT 200", flush=True)+except HTTPException as exc:+ print("RESULT", exc.status_code, exc.detail, flush=True)+'''++[email protected]("linux", "macos")+def test_gateway_boot_never_waits_on_a_slow_integrity_scan(tmp_path):+ """alt-glitch C1: a browse scanning a large lagging store must not hold gateway.lock. A gateway+ booting mid-scan takes the lock at once; the browse then sees the owner and writes nothing."""+ import os+ import subprocess+ import sys+ import time++ from gateway.runtime_ownership import ProfileOwnership++ home = _profile_home(tmp_path)+ path = _lagging_store_with_history(home, rows=50)+ before = _file_state(path)+ browse = subprocess.Popen([sys.executable, "-c", _SLOW_SCAN_BROWSE, str(path), "4"], env=os.environ,+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)+ assert browse.stdout.readline().strip() == "started"+ time.sleep(1.5) # inside the 4 s scan+ booting = ProfileOwnership()+ try:+ started = time.monotonic()+ booting.reserve([home]) # no boot wait: the lock must simply be free+ took = time.monotonic() - started+ out = browse.communicate(timeout=60)[0]+ finally:+ booting.close()+ assert took < 0.5+ result = next(line for line in out.splitlines() if line.startswith("RESULT"))+ assert result.startswith("RESULT 503"), out+ assert "running gateway owns it" in result+ assert _file_state(path) == before+++class _SlowProgress:+ """A real sqlite connection whose VM runs slowly, chaining to the code's own progress handler."""++ def __init__(self, conn):+ self._conn = conn+ self.set_progress_handler(None, 0)++ def __getattr__(self, name):+ return getattr(self._conn, name)++ def set_progress_handler(self, handler, n):+ import time++ def slow():+ time.sleep(0.01)+ return handler() if handler is not None else 0+ return self._conn.set_progress_handler(slow, 5)++ def close(self):+ return self._conn.close()+++def test_slow_integrity_scan_is_bounded_and_not_repeated_on_every_poll(tmp_path, monkeypatch):+ """alt-glitch C1: the pre-lock scan still has a deadline (retryable 503, nothing written), and a+ sidebar polling the same unchanged store does not rescan the whole file on every request."""+ import hermes_cli.sqlite_safe_read as ssr+ import hermes_state_reconcile as reconcile++ path = _lagging_store_with_history(_profile_home(tmp_path), rows=400)+ before = _file_state(path)+ original = ssr.connect_tracked+ monkeypatch.setattr(ssr, "connect_tracked", lambda p, *a, **kw: (+ _SlowProgress(original(p, *a, **kw)) if "mode=ro" in str(p) else original(p, *a, **kw)))+ monkeypatch.setattr(reconcile, "BROWSE_INTEGRITY_BUDGET_S", 0.3)+ scans = []+ real_scan = reconcile._integrity_problem+ monkeypatch.setattr(reconcile, "_integrity_problem", lambda p, b: scans.append(1) or real_scan(p, b))+ for _ in range(3):+ with pytest.raises(HTTPException) as caught:+ _open_session_db_at_path(path, read_only=True)+ assert caught.value.status_code == 503+ assert "nothing was changed" in caught.value.detail and "Retry" in caught.value.detail+ assert scans == [1]+ assert _file_state(path) == before