Skip to content

The agent that grows with you

fix(sessions): browse integrity scan runs before gateway.lock, never under it

Teknium7ddf40fee8ac
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py
index 9edbe3b75b3..53413907080 100644
--- a/gateway/run_bootstrap.py
+++ b/gateway/run_bootstrap.py
@@ -545,10 +545,11 @@ def _start_gateway_resolve_terminal_cwd() -> None:
os.environ["TERMINAL_CWD"] = resolved_cwd
-# A browse that catches a lagging state.db up holds the profile's gateway.lock for at most its
-# execution budget (hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S: integrity check + DDL, cut off
-# by a sqlite progress handler). A gateway booting in that window waits it out, with margin for
-# lock publication and the drift probes, instead of exiting as if another gateway owned the profile.
+# A browse that catches a lagging state.db up holds the profile's gateway.lock only for the drift
+# re-check and the additive DDL, cut off at hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S by a
+# sqlite progress handler. Its integrity scan (which scales with file size) runs BEFORE the lock, so
+# a large store never extends this window. A gateway booting in it waits it out, with margin for lock
+# publication and the drift probes, instead of exiting as if another gateway owned the profile.
def _browse_reconcile_reserve_wait_s() -> float:
from hermes_state_reconcile import BROWSE_RECONCILE_BUDGET_S
return BROWSE_RECONCILE_BUDGET_S + 2.0
diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.py
index 1f91b82bfb8..28c91e24270 100644
--- a/hermes_state_reconcile.py
+++ b/hermes_state_reconcile.py
@@ -8,6 +8,7 @@ surgery (:func:`repair_declarative_schema`). Neither runs while a gateway owns t
import contextlib
import logging
import sqlite3
+import threading
from contextlib import closing
from pathlib import Path
@@ -104,12 +105,15 @@ def _reconcile_locked(path, backup, report):
# profile's gateway runtime lock. When a gateway owns the store it is the only writer: refuse
# by name with the remedy instead of becoming a second one.
#
-# The lock is held for at most BROWSE_RECONCILE_BUDGET_S: one monotonic deadline, starting once the
-# lock is held, covers the integrity check and the DDL transaction. A sqlite progress handler
-# interrupts either one past the deadline (the DDL rolls back) and browse answers a retryable 503.
-# Busy waiting does not run the progress handler, so each connection's busy timeout is capped at
-# the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).
+# The full integrity scan is read-only and scales with file size, so it runs BEFORE the lock, on its
+# own budget (BROWSE_INTEGRITY_BUDGET_S); a gateway booting meanwhile simply takes the lock. The
+# scan is tied to the file's (size, mtime) signature: if the store changed by the time the lock is
+# held, the scan no longer vouches for it and nothing is written. Under the lock only the cheap drift
+# re-check and the DDL run, cut off by BROWSE_RECONCILE_BUDGET_S (a sqlite progress handler; the DDL
+# rolls back). Busy waiting does not run the progress handler, so each connection's busy timeout is
+# capped at the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).
BROWSE_RECONCILE_BUDGET_S = 3.0
+BROWSE_INTEGRITY_BUDGET_S = 20.0
_BUDGET_PROGRESS_OPS = 1000
@@ -263,6 +267,31 @@ def _wait_for_sibling_reconcile(path: Path, home: Path):
time.sleep(_BROWSE_SIBLING_POLL_S)
+def _store_signature(path: Path):
+ """(size, mtime_ns) of the main file and its WAL: any committed write changes one of them."""
+ out = []
+ for p in (path, path.with_name(path.name + '-wal')):
+ try:
+ st = p.stat()
+ out.append((st.st_size, st.st_mtime_ns))
+ except FileNotFoundError:
+ out.append(None)
+ return tuple(out)
+
+
+_scan_guard = threading.Lock()
+_scan_locks: dict[str, threading.Lock] = {}
+# Signatures whose integrity scan could not finish inside BROWSE_INTEGRITY_BUDGET_S. A sidebar polls
+# every few seconds; rescanning a multi-GB file on each poll only burns I/O for the same answer.
+_scan_too_slow: dict[str, tuple] = {}
+
+
+def _path_scan_lock(path: Path) -> threading.Lock:
+ key = str(path.resolve())
+ with _scan_guard:
+ return _scan_locks.setdefault(key, threading.Lock())
+
+
def reconcile_drift_for_browse(path: Path) -> bool:
"""Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.
@@ -271,6 +300,7 @@ def reconcile_drift_for_browse(path: Path) -> bool:
when the lagging store is also damaged. Never touches a store missing its history tables:
recreating them empty would hide lost sessions."""
from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance, live_runtime_owner
+ from hermes_state_health import STORAGE_CORRUPT, mark_storage_corrupt, storage_corrupt_reason, storage_state
reason = schema_drift_reason(path)
if reason is None:
@@ -291,41 +321,76 @@ def reconcile_drift_for_browse(path: Path) -> bool:
f"Session store for profile '{label}' is behind this Hermes version ({drift}), and a "
f'running gateway owns it, so browsing will not write to it. Restart that gateway '
f'(`{remedy}`) to bring the store up to date.', owned=True)
+
+ def too_slow(what):
+ return DeclarativeDriftError(
+ f"Session store for profile '{label}' is behind this Hermes version ({reason}) and could "
+ f'not be {what} while browsing; nothing was changed and the list was not cleared. Retry, '
+ f"or start this profile's gateway (`{remedy}`) or run `hermes sessions repair` to bring "
+ 'it up to date.')
+
+ # A gateway already owns it: never scan or write, refuse by name at once. Probe the flock itself
+ # (a record left by a released holder still names a live PID), holding it for no work at all.
try:
with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):
- budget = _Budget(BROWSE_RECONCILE_BUDGET_S)
- # Re-check under the lock: a gateway that started and released between probes
- # (or a sibling browse) may already have caught the store up.
- if schema_drift_reason(path) is None:
- return False
- # A failing column probe hides damage elsewhere in the file: additive DDL is not
- # corruption recovery, so never write into a store that is not verified clean.
- if problem := _integrity_problem(path, budget):
- raise CorruptStoreError(problem)
- try:
- _apply_additive_schema(path, budget)
- except DeclarativeDriftError as exc:
- raise DeclarativeDriftError(
- f"Session store for profile '{label}' is behind this Hermes version and could not "
- f'be caught up ({exc}); nothing was changed. Run `hermes sessions repair` for that '
- 'profile.') from exc
- except BrowseBudgetExceeded as exc:
- raise DeclarativeDriftError(
- f"Session store for profile '{label}' is behind this Hermes version ({reason}) and could "
- f'not be verified and caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s while browsing; '
- f'nothing was changed and the list was not cleared. Retry, or start this profile\'s '
- f'gateway (`{remedy}`) or run `hermes sessions repair` to bring it up to date.') from exc
+ pass
except OwnershipConflict as exc:
if live_runtime_owner(home) is not None:
raise owned(reason) from exc
- # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).
- if (pending := _wait_for_sibling_reconcile(path, home)) is None:
+ # Already latched corrupt in this process: do not rescan a damaged file on every poll.
+ if storage_state(path) == STORAGE_CORRUPT:
+ raise CorruptStoreError(storage_corrupt_reason(path) or 'latched corrupt')
+ key = str(path.resolve())
+ with _path_scan_lock(path): # one scan per store per process; concurrent polls wait for it
+ if schema_drift_reason(path) is None:
return False
- if live_runtime_owner(home) is not None:
- raise owned(pending) from exc
- raise DeclarativeDriftError(
- f"Session store for profile '{label}' is being brought up to date by another Hermes "
- f'process ({pending}). Retry in a moment; the list was not cleared.') from exc
+ signature = _store_signature(path)
+ if _scan_too_slow.get(key) == signature:
+ raise too_slow(f'verified within {BROWSE_INTEGRITY_BUDGET_S:.0f}s')
+ # Outside gateway.lock: read-only, may be slow on a large store, never blocks a gateway boot.
+ try:
+ problem = _integrity_problem(path, _Budget(BROWSE_INTEGRITY_BUDGET_S))
+ except BrowseBudgetExceeded as exc:
+ _scan_too_slow[key] = signature
+ raise too_slow(f'verified within {BROWSE_INTEGRITY_BUDGET_S:.0f}s') from exc
+ if problem:
+ # A failing column probe hides damage elsewhere in the file: additive DDL is not
+ # corruption recovery, so never write into a store that is not verified clean.
+ mark_storage_corrupt(path, problem)
+ raise CorruptStoreError(problem)
+ try:
+ with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):
+ budget = _Budget(BROWSE_RECONCILE_BUDGET_S)
+ # Re-check under the lock: a gateway that started and released between probes
+ # (or a sibling browse) may already have caught the store up.
+ if schema_drift_reason(path) is None:
+ return False
+ if _store_signature(path) != signature:
+ # Written since the scan (a gateway that came and went): the scan no longer
+ # vouches for these bytes. Refuse; the next browse scans the new state.
+ raise DeclarativeDriftError(
+ f"Session store for profile '{label}' changed while browsing verified it "
+ f'({reason}); nothing was changed. Retry in a moment; the list was not cleared.')
+ try:
+ _apply_additive_schema(path, budget)
+ except DeclarativeDriftError as exc:
+ raise DeclarativeDriftError(
+ f"Session store for profile '{label}' is behind this Hermes version and could not "
+ f'be caught up ({exc}); nothing was changed. Run `hermes sessions repair` for that '
+ 'profile.') from exc
+ except BrowseBudgetExceeded as exc:
+ raise too_slow(f'caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s') from exc
+ except OwnershipConflict as exc:
+ if live_runtime_owner(home) is not None:
+ raise owned(reason) from exc
+ # No gateway: the holder is a sibling browse's own reconcile (or other short maintenance).
+ if (pending := _wait_for_sibling_reconcile(path, home)) is None:
+ return False
+ if live_runtime_owner(home) is not None:
+ raise owned(pending) from exc
+ raise DeclarativeDriftError(
+ f"Session store for profile '{label}' is being brought up to date by another Hermes "
+ f'process ({pending}). Retry in a moment; the list was not cleared.') from exc
if after := schema_drift_reason(path):
raise DeclarativeDriftError(
f"Session store for profile '{label}' is behind this Hermes version and could not be "
diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.py
index 96612059f34..bfd5cde1bf2 100644
--- a/tests/hermes_cli/test_browse_schema_drift.py
+++ b/tests/hermes_cli/test_browse_schema_drift.py
@@ -546,6 +546,7 @@ def slowed(path, *a, **kw):
conn = original(path, *a, **kw)
read_only = "mode=ro" in str(path)
if holding_lock() and (phase == "integrity") == read_only:
+ print("slowed", flush=True) # this process holds gateway.lock and the slow phase begins
return SlowConnection(conn)
return conn
@@ -561,11 +562,12 @@ except HTTPException as exc:
@pytest.mark.platforms("linux", "macos")
[email protected]("phase", ["integrity", "ddl"])
[email protected]("phase", ["ddl"])
def test_slow_browse_reconcile_gives_up_inside_its_budget_and_boot_waits_it_out(tmp_path, phase):
- """R2-2: the busy timeout bounds lock contention, not SQL execution. A slow integrity scan or
- DDL must be interrupted at the browse budget (rolled back, retryable 503), and gateway boot's
- wait must outlast that budget, so a slow browse never turns into a startup refusal."""
+ """R2-2: the busy timeout bounds lock contention, not SQL execution. A slow DDL must be
+ interrupted at the browse budget (rolled back, retryable 503), and gateway boot's wait must
+ outlast that budget, so a slow browse never turns into a startup refusal. (The integrity scan
+ no longer runs under the lock: see test_gateway_boot_never_waits_on_a_slow_integrity_scan.)"""
import os
import subprocess
import sys
@@ -584,10 +586,7 @@ def test_slow_browse_reconcile_gives_up_inside_its_budget_and_boot_waits_it_out(
browse = subprocess.Popen([sys.executable, "-c", _SLOW_BROWSE, str(path), phase], env=os.environ,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
assert browse.stdout.readline().strip() == "started"
- deadline = time.monotonic() + 20
- while not (home / "gateway.lock").exists() or not (home / "gateway.lock").read_text():
- assert time.monotonic() < deadline and browse.poll() is None
- time.sleep(0.01)
+ assert browse.stdout.readline().strip() == "slowed", "the slow phase never ran under the lock"
booting = ProfileOwnership()
try:
bootstrap._reserve_waiting_out_browse_reconcile(booting, [home]) # default wait: must not raise
@@ -653,3 +652,149 @@ def test_failed_column_reconcile_rolls_back_every_addition(tmp_path):
assert "nothing was changed" in str(caught.value.detail)
assert _file_state(path) == before
assert "last_read_at" not in _session_columns(path)
+
+
+# ── Maintainer review round 3 ──
+
+
+def test_integrity_scan_runs_before_the_lock_is_taken(tmp_path, monkeypatch):
+ """alt-glitch C1 / pastels m3: the full integrity scan is read-only and scales with file size.
+ It must run BEFORE gateway.lock is reserved, so a booting gateway never waits on it."""
+ import hermes_state_reconcile as reconcile
+ from gateway.runtime_ownership import OwnershipConflict, ProfileOwnership
+
+ home = _profile_home(tmp_path)
+ path = _lagging_store_with_history(home, rows=10)
+ original = reconcile._integrity_problem
+ lock_free_during_scan = []
+
+ def spy(p, budget):
+ other = ProfileOwnership() # its own open file description: conflicts with any holder
+ try:
+ other.reserve([home])
+ lock_free_during_scan.append(True)
+ except OwnershipConflict:
+ lock_free_during_scan.append(False)
+ finally:
+ other.close()
+ return original(p, budget)
+
+ monkeypatch.setattr(reconcile, "_integrity_problem", spy)
+ _open_session_db_at_path(path, read_only=True).close()
+ assert lock_free_during_scan == [True]
+ assert "last_read_at" in _session_columns(path)
+
+
+_SLOW_SCAN_BROWSE = r'''
+import sys, time
+from pathlib import Path
+import hermes_cli.sqlite_safe_read as ssr
+from fastapi import HTTPException
+from hermes_cli.web_server_sessions import _open_session_db_at_path
+
+original = ssr.connect_tracked
+
+class SlowScan:
+ """A real read-only connection whose integrity scan runs slowly (a cold multi-GB store)."""
+ def __init__(self, conn):
+ self._conn = conn
+ def __getattr__(self, name):
+ return getattr(self._conn, name)
+ def execute(self, sql, *a):
+ if sql.startswith("PRAGMA integrity_check"):
+ time.sleep(float(sys.argv[2]))
+ return self._conn.execute(sql, *a)
+ def close(self):
+ return self._conn.close()
+
+def slowed(path, *a, **kw):
+ conn = original(path, *a, **kw)
+ return SlowScan(conn) if "mode=ro" in str(path) else conn
+
+ssr.connect_tracked = slowed
+print("started", flush=True)
+try:
+ _open_session_db_at_path(Path(sys.argv[1]), read_only=True).close()
+ print("RESULT 200", flush=True)
+except HTTPException as exc:
+ print("RESULT", exc.status_code, exc.detail, flush=True)
+'''
+
+
[email protected]("linux", "macos")
+def test_gateway_boot_never_waits_on_a_slow_integrity_scan(tmp_path):
+ """alt-glitch C1: a browse scanning a large lagging store must not hold gateway.lock. A gateway
+ booting mid-scan takes the lock at once; the browse then sees the owner and writes nothing."""
+ import os
+ import subprocess
+ import sys
+ import time
+
+ from gateway.runtime_ownership import ProfileOwnership
+
+ home = _profile_home(tmp_path)
+ path = _lagging_store_with_history(home, rows=50)
+ before = _file_state(path)
+ browse = subprocess.Popen([sys.executable, "-c", _SLOW_SCAN_BROWSE, str(path), "4"], env=os.environ,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+ assert browse.stdout.readline().strip() == "started"
+ time.sleep(1.5) # inside the 4 s scan
+ booting = ProfileOwnership()
+ try:
+ started = time.monotonic()
+ booting.reserve([home]) # no boot wait: the lock must simply be free
+ took = time.monotonic() - started
+ out = browse.communicate(timeout=60)[0]
+ finally:
+ booting.close()
+ assert took < 0.5
+ result = next(line for line in out.splitlines() if line.startswith("RESULT"))
+ assert result.startswith("RESULT 503"), out
+ assert "running gateway owns it" in result
+ assert _file_state(path) == before
+
+
+class _SlowProgress:
+ """A real sqlite connection whose VM runs slowly, chaining to the code's own progress handler."""
+
+ def __init__(self, conn):
+ self._conn = conn
+ self.set_progress_handler(None, 0)
+
+ def __getattr__(self, name):
+ return getattr(self._conn, name)
+
+ def set_progress_handler(self, handler, n):
+ import time
+
+ def slow():
+ time.sleep(0.01)
+ return handler() if handler is not None else 0
+ return self._conn.set_progress_handler(slow, 5)
+
+ def close(self):
+ return self._conn.close()
+
+
+def test_slow_integrity_scan_is_bounded_and_not_repeated_on_every_poll(tmp_path, monkeypatch):
+ """alt-glitch C1: the pre-lock scan still has a deadline (retryable 503, nothing written), and a
+ sidebar polling the same unchanged store does not rescan the whole file on every request."""
+ import hermes_cli.sqlite_safe_read as ssr
+ import hermes_state_reconcile as reconcile
+
+ path = _lagging_store_with_history(_profile_home(tmp_path), rows=400)
+ before = _file_state(path)
+ original = ssr.connect_tracked
+ monkeypatch.setattr(ssr, "connect_tracked", lambda p, *a, **kw: (
+ _SlowProgress(original(p, *a, **kw)) if "mode=ro" in str(p) else original(p, *a, **kw)))
+ monkeypatch.setattr(reconcile, "BROWSE_INTEGRITY_BUDGET_S", 0.3)
+ scans = []
+ real_scan = reconcile._integrity_problem
+ monkeypatch.setattr(reconcile, "_integrity_problem", lambda p, b: scans.append(1) or real_scan(p, b))
+ for _ in range(3):
+ with pytest.raises(HTTPException) as caught:
+ _open_session_db_at_path(path, read_only=True)
+ assert caught.value.status_code == 503
+ assert "nothing was changed" in caught.value.detail and "Retry" in caught.value.detail
+ assert scans == [1]
+ assert _file_state(path) == before