Skip to content

The agent that grows with you

fix(sessions): bound browse catch-up with a real budget and a full integrity check

Teknium4220faa556a2
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py
index 475f0348bc2..9edbe3b75b3 100644
--- a/gateway/run_bootstrap.py
+++ b/gateway/run_bootstrap.py
@@ -546,17 +546,22 @@ def _start_gateway_resolve_terminal_cwd() -> None:
# A browse that catches a lagging state.db up holds the profile's gateway.lock for at most its
-# sqlite busy budget (hermes_state_reconcile._BROWSE_RECONCILE_BUSY_S, 2 s). A gateway booting in
-# that window waits it out instead of exiting as if another gateway owned the profile.
-_BROWSE_RECONCILE_RESERVE_WAIT_S = 5.0
+# execution budget (hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S: integrity check + DDL, cut off
+# by a sqlite progress handler). A gateway booting in that window waits it out, with margin for
+# lock publication and the drift probes, instead of exiting as if another gateway owned the profile.
+def _browse_reconcile_reserve_wait_s() -> float:
+ from hermes_state_reconcile import BROWSE_RECONCILE_BUDGET_S
+ return BROWSE_RECONCILE_BUDGET_S + 2.0
-def _reserve_waiting_out_browse_reconcile(ownership, homes, *, wait_s: float = _BROWSE_RECONCILE_RESERVE_WAIT_S):
+def _reserve_waiting_out_browse_reconcile(ownership, homes, *, wait_s: float | None = None):
"""Reserve *homes*, waiting (bounded) while a contender is a browse reconcile or has not yet
published its lock record (flock taken, record unwritten). Only a positively identified holder
that is not browse maintenance refuses at once; the flock exclusion itself is never weakened."""
from gateway.runtime_ownership import OwnershipConflict, lock_record_unpublished, maintenance_holder
from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
+ if wait_s is None:
+ wait_s = _browse_reconcile_reserve_wait_s()
deadline = time.monotonic() + wait_s
logged = False
while True:
diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.py
index 5f9d0f0b803..d31cfb6a797 100644
--- a/hermes_state_reconcile.py
+++ b/hermes_state_reconcile.py
@@ -103,7 +103,14 @@ def _reconcile_locked(path, backup, report):
# Catch it up ONCE, additive DDL only, in one short transaction, and only while holding that
# profile's gateway runtime lock. When a gateway owns the store it is the only writer: refuse
# by name with the remedy instead of becoming a second one.
-_BROWSE_RECONCILE_BUSY_S = 2.0
+#
+# The lock is held for at most BROWSE_RECONCILE_BUDGET_S: one monotonic deadline, starting once the
+# lock is held, covers the integrity check and the DDL transaction. A sqlite progress handler
+# interrupts either one past the deadline (the DDL rolls back) and browse answers a retryable 503.
+# Busy waiting does not run the progress handler, so each connection's busy timeout is capped at
+# the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).
+BROWSE_RECONCILE_BUDGET_S = 3.0
+_BUDGET_PROGRESS_OPS = 1000
class DeclarativeDriftError(RuntimeError):
@@ -128,20 +135,56 @@ def _owner_remedy(home: Path) -> tuple[str, str]:
return label or str(home), 'hermes gateway restart'
-def _apply_additive_schema(path: Path) -> None:
+class BrowseBudgetExceeded(RuntimeError):
+ """The browse reconcile could not finish inside BROWSE_RECONCILE_BUDGET_S; nothing was written."""
+
+
+class _Budget:
+ """One monotonic deadline for everything a browse does while holding ``gateway.lock``."""
+
+ def __init__(self, seconds: float):
+ import time
+ self._clock = time.monotonic
+ self.deadline = self._clock() + seconds
+ self.expired = False
+
+ def remaining(self) -> float:
+ return max(self.deadline - self._clock(), 0.0)
+
+ def busy_timeout(self) -> float:
+ return max(self.remaining(), 0.05)
+
+ def arm(self, conn) -> None:
+ def progress():
+ if self._clock() >= self.deadline:
+ self.expired = True
+ return 1 # sqlite raises OperationalError('interrupted'); the caller rolls back
+ return 0
+ conn.set_progress_handler(progress, _BUDGET_PROGRESS_OPS)
+
+ def check(self) -> None:
+ if self.expired or self.remaining() <= 0:
+ self.expired = True
+ raise BrowseBudgetExceeded('browse schema catch-up exceeded its time budget')
+
+
+def _apply_additive_schema(path: Path, budget: _Budget | None = None) -> None:
"""Missing SCHEMA_SQL tables and columns, in ONE ``BEGIN IMMEDIATE`` transaction.
No data migration, no FTS work, no schema_version stamp (that stays the owner's: the
version gates data migrations this path does not run). Every statement is additive and
- idempotent, so a failure rolls the store back to exactly what it was."""
+ idempotent, so a failure (or an expired budget) rolls the store back to exactly what it was."""
from hermes_cli.sqlite_safe_read import connect_tracked
from hermes_state_common import SCHEMA_SQL
from hermes_state_repair import _reapply_durability_barriers
from hermes_state_schema import SessionSchemaMixin
- with closing(connect_tracked(path, tracking_path=path, timeout=_BROWSE_RECONCILE_BUSY_S,
+ budget = budget or _Budget(BROWSE_RECONCILE_BUDGET_S)
+ budget.check()
+ with closing(connect_tracked(path, tracking_path=path, timeout=budget.busy_timeout(),
isolation_level=None)) as conn:
_reapply_durability_barriers(conn)
+ budget.arm(conn)
conn.execute('BEGIN IMMEDIATE')
try:
cursor = conn.cursor()
@@ -150,10 +193,13 @@ def _apply_additive_schema(path: Path) -> None:
shim._reconcile_columns(cursor)
SessionSchemaMixin._execute_ddl_script_transactional(cursor, SCHEMA_SQL)
shim._reconcile_columns(cursor) # tables the script just created are complete; no-op
+ budget.check()
conn.execute('COMMIT')
- except BaseException:
+ except BaseException as exc:
with contextlib.suppress(sqlite3.Error):
conn.execute('ROLLBACK')
+ if budget.expired and isinstance(exc, sqlite3.OperationalError):
+ raise BrowseBudgetExceeded('browse schema catch-up exceeded its time budget') from exc
raise
@@ -166,30 +212,39 @@ class CorruptStoreError(sqlite3.DatabaseError):
own malformed-image error so every caller maps it to the corrupt-store status (``hermes doctor``)."""
def __init__(self, detail: str):
- super().__init__(f'database disk image is malformed (quick_check: {detail})')
+ super().__init__(f'database disk image is malformed (integrity_check: {detail})')
-def _quick_check_problem(path: Path):
- """``PRAGMA quick_check`` on a read-only connection: None when ``ok``, else the first problems."""
+def _integrity_problem(path: Path, budget: _Budget):
+ """``PRAGMA integrity_check`` on a read-only connection under *budget*: None when ``ok``, else the
+ first problems. Full integrity_check, not quick_check: quick_check skips index-vs-table content,
+ so an index with the right entry count but the wrong keys passes it. Raises
+ :class:`BrowseBudgetExceeded` when the scan cannot finish in time (fail closed: no write)."""
from hermes_cli.sqlite_safe_read import connect_tracked
+ budget.check()
try:
- with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path)) as conn:
- rows = conn.execute('PRAGMA quick_check').fetchall()
+ with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path,
+ timeout=budget.busy_timeout())) as conn:
+ budget.arm(conn)
+ rows = conn.execute('PRAGMA integrity_check(3)').fetchall()
except sqlite3.DatabaseError as exc:
+ if budget.expired:
+ raise BrowseBudgetExceeded('integrity check exceeded the browse time budget') from exc
from hermes_state_errors import is_malformed_db_error
if is_malformed_db_error(exc):
return str(exc)
raise
+ budget.check()
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != 'ok']
return '; '.join(problems[:3]) or None
def _wait_for_sibling_reconcile(path: Path, home: Path):
"""A sibling browse (another thread or process) holds the lock for its own reconcile: wait at
- most the busy budget for it. None once the store is current, else the remaining drift."""
+ at most the sibling's own execution budget. None once the store is current, else the remaining drift."""
import time
from gateway.runtime_ownership import live_runtime_owner
- deadline = time.monotonic() + _BROWSE_RECONCILE_BUSY_S
+ deadline = time.monotonic() + BROWSE_RECONCILE_BUDGET_S + 0.5
while True:
if (reason := schema_drift_reason(path)) is None:
return None
@@ -228,15 +283,22 @@ def reconcile_drift_for_browse(path: Path) -> bool:
f'(`{remedy}`) to bring the store up to date.', owned=True)
try:
with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):
+ budget = _Budget(BROWSE_RECONCILE_BUDGET_S)
# Re-check under the lock: a gateway that started and released between probes
# (or a sibling browse) may already have caught the store up.
if schema_drift_reason(path) is None:
return False
# A failing column probe hides damage elsewhere in the file: additive DDL is not
- # corruption recovery, so never write into a store that is not clean.
- if problem := _quick_check_problem(path):
+ # corruption recovery, so never write into a store that is not verified clean.
+ if problem := _integrity_problem(path, budget):
raise CorruptStoreError(problem)
- _apply_additive_schema(path)
+ _apply_additive_schema(path, budget)
+ except BrowseBudgetExceeded as exc:
+ raise DeclarativeDriftError(
+ f"Session store for profile '{label}' is behind this Hermes version ({reason}) and could "
+ f'not be verified and caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s while browsing; '
+ f'nothing was changed and the list was not cleared. Retry, or start this profile\'s '
+ f'gateway (`{remedy}`) or run `hermes sessions repair` to bring it up to date.') from exc
except OwnershipConflict as exc:
if live_runtime_owner(home) is not None:
raise owned(reason) from exc
diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.py
index d0d25f01602..dda0c318c87 100644
--- a/tests/hermes_cli/test_browse_schema_drift.py
+++ b/tests/hermes_cli/test_browse_schema_drift.py
@@ -506,3 +506,130 @@ def test_gateway_boot_waits_while_the_holder_has_not_published_its_record(tmp_pa
holder.kill()
holder.wait()
+
+_SLOW_BROWSE = r'''
+import json, os, sqlite3, sys, time
+from pathlib import Path
+import hermes_cli.sqlite_safe_read as ssr
+from fastapi import HTTPException
+from hermes_cli.web_server_sessions import _open_session_db_at_path
+
+phase = sys.argv[2]
+original = ssr.connect_tracked
+
+class SlowConnection:
+ """A real sqlite connection whose VM runs slowly (stand-in for a cold multi-GB disk). The
+ slowdown is installed here, whether or not the code under test sets its own progress handler,
+ and chains to that handler when it does."""
+ def __init__(self, conn):
+ self._conn = conn
+ self.set_progress_handler(None, 0)
+ def __getattr__(self, name):
+ return getattr(self._conn, name)
+ def set_progress_handler(self, handler, n):
+ def slow():
+ time.sleep(0.01)
+ return handler() if handler is not None else 0
+ return self._conn.set_progress_handler(slow, 5)
+ def close(self):
+ return self._conn.close()
+
+def holding_lock():
+ """Only connections opened while THIS process holds gateway.lock are slowed."""
+ lock = Path(sys.argv[1]).with_name("gateway.lock")
+ try:
+ return json.loads(lock.read_text()).get("pid") == os.getpid()
+ except (OSError, ValueError):
+ return False
+
+def slowed(path, *a, **kw):
+ conn = original(path, *a, **kw)
+ read_only = "mode=ro" in str(path)
+ if holding_lock() and (phase == "integrity") == read_only:
+ return SlowConnection(conn)
+ return conn
+
+ssr.connect_tracked = slowed
+started = time.monotonic()
+print("started", flush=True)
+try:
+ _open_session_db_at_path(Path(sys.argv[1]), read_only=True).close()
+ print("RESULT 200", time.monotonic() - started, flush=True)
+except HTTPException as exc:
+ print("RESULT", exc.status_code, time.monotonic() - started, exc.detail, flush=True)
+'''
+
+
[email protected]("linux", "macos")
[email protected]("phase", ["integrity", "ddl"])
+def test_slow_browse_reconcile_gives_up_inside_its_budget_and_boot_waits_it_out(tmp_path, phase):
+ """R2-2: the busy timeout bounds lock contention, not SQL execution. A slow integrity scan or
+ DDL must be interrupted at the browse budget (rolled back, retryable 503), and gateway boot's
+ wait must outlast that budget, so a slow browse never turns into a startup refusal."""
+ import os
+ import subprocess
+ import sys
+ import time
+
+ import gateway.run_bootstrap as bootstrap
+ import hermes_state_reconcile as reconcile
+ from gateway.runtime_ownership import ProfileOwnership
+
+ budget_s = getattr(reconcile, "BROWSE_RECONCILE_BUDGET_S", 2.0)
+ wait_s = getattr(bootstrap, "_browse_reconcile_reserve_wait_s", lambda: 5.0)()
+ assert wait_s > budget_s + 1
+ home = _profile_home(tmp_path)
+ path = _lagging_store_with_history(home, rows=400)
+ before = _file_state(path)
+ browse = subprocess.Popen([sys.executable, "-c", _SLOW_BROWSE, str(path), phase], env=os.environ,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+ assert browse.stdout.readline().strip() == "started"
+ deadline = time.monotonic() + 20
+ while not (home / "gateway.lock").exists() or not (home / "gateway.lock").read_text():
+ assert time.monotonic() < deadline and browse.poll() is None
+ time.sleep(0.01)
+ booting = ProfileOwnership()
+ try:
+ bootstrap._reserve_waiting_out_browse_reconcile(booting, [home]) # default wait: must not raise
+ assert booting.owns(home)
+ finally:
+ booting.close()
+ out = browse.communicate(timeout=60)[0]
+ result = next(line for line in out.splitlines() if line.startswith("RESULT"))
+ _, status, elapsed, *detail = result.split(" ", 3)
+ assert status == "503", out
+ assert float(elapsed) < budget_s + 2, out
+ assert "nothing was changed" in " ".join(detail)
+ assert _file_state(path) == before
+ assert "last_read_at" not in _session_columns(path)
+
+
+def test_equal_cardinality_corrupt_index_plus_drift_is_not_written(tmp_path):
+ """R2-3: quick_check does not compare index keys with the table, so two equal-size indexes
+ with swapped root pages pass it. Browse must still refuse as corrupt and write nothing."""
+ path = _lagging_store_with_history(_profile_home(tmp_path), rows=10)
+ with closing(sqlite3.connect(path)) as conn:
+ conn.execute("CREATE TABLE probe_other (v TEXT)")
+ conn.execute("CREATE INDEX probe_other_index ON probe_other(v)")
+ conn.execute("INSERT INTO probe_other(v) VALUES ('wrong-source')")
+ conn.commit()
+ target = conn.execute("SELECT rootpage FROM sqlite_master WHERE name='idx_sessions_source'").fetchone()[0]
+ other = conn.execute("SELECT rootpage FROM sqlite_master WHERE name='probe_other_index'").fetchone()[0]
+ conn.execute("PRAGMA writable_schema=ON")
+ conn.execute("UPDATE sqlite_master SET rootpage=? WHERE name='idx_sessions_source'", (other,))
+ conn.execute("UPDATE sqlite_master SET rootpage=? WHERE name='probe_other_index'", (target,))
+ conn.execute("PRAGMA writable_schema=OFF")
+ conn.commit()
+ conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+ with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn:
+ assert conn.execute("PRAGMA quick_check").fetchall() == [("ok",)]
+ assert conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok"
+ before = _file_state(path)
+
+ with pytest.raises(HTTPException) as caught:
+ _open_session_db_at_path(path, read_only=True).close()
+ assert caught.value.status_code == 503
+ assert caught.value.detail["error"] == "state_db_corrupt"
+ assert _file_state(path) == before
+ assert "last_read_at" not in _session_columns(path)
+