The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(sessions): bound browse catch-up with a real budget and a full integrity check
Teknium4220faa556a2
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.pyindex 475f0348bc2..9edbe3b75b3 100644--- a/gateway/run_bootstrap.py+++ b/gateway/run_bootstrap.py@@ -546,17 +546,22 @@ def _start_gateway_resolve_terminal_cwd() -> None:# A browse that catches a lagging state.db up holds the profile's gateway.lock for at most its-# sqlite busy budget (hermes_state_reconcile._BROWSE_RECONCILE_BUSY_S, 2 s). A gateway booting in-# that window waits it out instead of exiting as if another gateway owned the profile.-_BROWSE_RECONCILE_RESERVE_WAIT_S = 5.0+# execution budget (hermes_state_reconcile.BROWSE_RECONCILE_BUDGET_S: integrity check + DDL, cut off+# by a sqlite progress handler). A gateway booting in that window waits it out, with margin for+# lock publication and the drift probes, instead of exiting as if another gateway owned the profile.+def _browse_reconcile_reserve_wait_s() -> float:+ from hermes_state_reconcile import BROWSE_RECONCILE_BUDGET_S+ return BROWSE_RECONCILE_BUDGET_S + 2.0-def _reserve_waiting_out_browse_reconcile(ownership, homes, *, wait_s: float = _BROWSE_RECONCILE_RESERVE_WAIT_S):+def _reserve_waiting_out_browse_reconcile(ownership, homes, *, wait_s: float | None = None):"""Reserve *homes*, waiting (bounded) while a contender is a browse reconcile or has not yetpublished its lock record (flock taken, record unwritten). Only a positively identified holderthat is not browse maintenance refuses at once; the flock exclusion itself is never weakened."""from gateway.runtime_ownership import OwnershipConflict, lock_record_unpublished, maintenance_holderfrom hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE+ if wait_s is None:+ wait_s = _browse_reconcile_reserve_wait_s()deadline = time.monotonic() + wait_slogged = Falsewhile True:diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.pyindex 5f9d0f0b803..d31cfb6a797 100644--- a/hermes_state_reconcile.py+++ b/hermes_state_reconcile.py@@ -103,7 +103,14 @@ def _reconcile_locked(path, backup, report):# Catch it up ONCE, additive DDL only, in one short transaction, and only while holding that# profile's gateway runtime lock. When a gateway owns the store it is the only writer: refuse# by name with the remedy instead of becoming a second one.-_BROWSE_RECONCILE_BUSY_S = 2.0+#+# The lock is held for at most BROWSE_RECONCILE_BUDGET_S: one monotonic deadline, starting once the+# lock is held, covers the integrity check and the DDL transaction. A sqlite progress handler+# interrupts either one past the deadline (the DDL rolls back) and browse answers a retryable 503.+# Busy waiting does not run the progress handler, so each connection's busy timeout is capped at+# the remaining budget. Gateway boot waits this budget out (gateway.run_bootstrap).+BROWSE_RECONCILE_BUDGET_S = 3.0+_BUDGET_PROGRESS_OPS = 1000class DeclarativeDriftError(RuntimeError):@@ -128,20 +135,56 @@ def _owner_remedy(home: Path) -> tuple[str, str]:return label or str(home), 'hermes gateway restart'-def _apply_additive_schema(path: Path) -> None:+class BrowseBudgetExceeded(RuntimeError):+ """The browse reconcile could not finish inside BROWSE_RECONCILE_BUDGET_S; nothing was written."""+++class _Budget:+ """One monotonic deadline for everything a browse does while holding ``gateway.lock``."""++ def __init__(self, seconds: float):+ import time+ self._clock = time.monotonic+ self.deadline = self._clock() + seconds+ self.expired = False++ def remaining(self) -> float:+ return max(self.deadline - self._clock(), 0.0)++ def busy_timeout(self) -> float:+ return max(self.remaining(), 0.05)++ def arm(self, conn) -> None:+ def progress():+ if self._clock() >= self.deadline:+ self.expired = True+ return 1 # sqlite raises OperationalError('interrupted'); the caller rolls back+ return 0+ conn.set_progress_handler(progress, _BUDGET_PROGRESS_OPS)++ def check(self) -> None:+ if self.expired or self.remaining() <= 0:+ self.expired = True+ raise BrowseBudgetExceeded('browse schema catch-up exceeded its time budget')+++def _apply_additive_schema(path: Path, budget: _Budget | None = None) -> None:"""Missing SCHEMA_SQL tables and columns, in ONE ``BEGIN IMMEDIATE`` transaction.No data migration, no FTS work, no schema_version stamp (that stays the owner's: theversion gates data migrations this path does not run). Every statement is additive and- idempotent, so a failure rolls the store back to exactly what it was."""+ idempotent, so a failure (or an expired budget) rolls the store back to exactly what it was."""from hermes_cli.sqlite_safe_read import connect_trackedfrom hermes_state_common import SCHEMA_SQLfrom hermes_state_repair import _reapply_durability_barriersfrom hermes_state_schema import SessionSchemaMixin- with closing(connect_tracked(path, tracking_path=path, timeout=_BROWSE_RECONCILE_BUSY_S,+ budget = budget or _Budget(BROWSE_RECONCILE_BUDGET_S)+ budget.check()+ with closing(connect_tracked(path, tracking_path=path, timeout=budget.busy_timeout(),isolation_level=None)) as conn:_reapply_durability_barriers(conn)+ budget.arm(conn)conn.execute('BEGIN IMMEDIATE')try:cursor = conn.cursor()@@ -150,10 +193,13 @@ def _apply_additive_schema(path: Path) -> None:shim._reconcile_columns(cursor)SessionSchemaMixin._execute_ddl_script_transactional(cursor, SCHEMA_SQL)shim._reconcile_columns(cursor) # tables the script just created are complete; no-op+ budget.check()conn.execute('COMMIT')- except BaseException:+ except BaseException as exc:with contextlib.suppress(sqlite3.Error):conn.execute('ROLLBACK')+ if budget.expired and isinstance(exc, sqlite3.OperationalError):+ raise BrowseBudgetExceeded('browse schema catch-up exceeded its time budget') from excraise@@ -166,30 +212,39 @@ class CorruptStoreError(sqlite3.DatabaseError):own malformed-image error so every caller maps it to the corrupt-store status (``hermes doctor``)."""def __init__(self, detail: str):- super().__init__(f'database disk image is malformed (quick_check: {detail})')+ super().__init__(f'database disk image is malformed (integrity_check: {detail})')-def _quick_check_problem(path: Path):- """``PRAGMA quick_check`` on a read-only connection: None when ``ok``, else the first problems."""+def _integrity_problem(path: Path, budget: _Budget):+ """``PRAGMA integrity_check`` on a read-only connection under *budget*: None when ``ok``, else the+ first problems. Full integrity_check, not quick_check: quick_check skips index-vs-table content,+ so an index with the right entry count but the wrong keys passes it. Raises+ :class:`BrowseBudgetExceeded` when the scan cannot finish in time (fail closed: no write)."""from hermes_cli.sqlite_safe_read import connect_tracked+ budget.check()try:- with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path)) as conn:- rows = conn.execute('PRAGMA quick_check').fetchall()+ with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path,+ timeout=budget.busy_timeout())) as conn:+ budget.arm(conn)+ rows = conn.execute('PRAGMA integrity_check(3)').fetchall()except sqlite3.DatabaseError as exc:+ if budget.expired:+ raise BrowseBudgetExceeded('integrity check exceeded the browse time budget') from excfrom hermes_state_errors import is_malformed_db_errorif is_malformed_db_error(exc):return str(exc)raise+ budget.check()problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != 'ok']return '; '.join(problems[:3]) or Nonedef _wait_for_sibling_reconcile(path: Path, home: Path):"""A sibling browse (another thread or process) holds the lock for its own reconcile: wait at- most the busy budget for it. None once the store is current, else the remaining drift."""+ at most the sibling's own execution budget. None once the store is current, else the remaining drift."""import timefrom gateway.runtime_ownership import live_runtime_owner- deadline = time.monotonic() + _BROWSE_RECONCILE_BUSY_S+ deadline = time.monotonic() + BROWSE_RECONCILE_BUDGET_S + 0.5while True:if (reason := schema_drift_reason(path)) is None:return None@@ -228,15 +283,22 @@ def reconcile_drift_for_browse(path: Path) -> bool:f'(`{remedy}`) to bring the store up to date.', owned=True)try:with exclusive_maintenance([home], purpose=BROWSE_RECONCILE_PURPOSE):+ budget = _Budget(BROWSE_RECONCILE_BUDGET_S)# Re-check under the lock: a gateway that started and released between probes# (or a sibling browse) may already have caught the store up.if schema_drift_reason(path) is None:return False# A failing column probe hides damage elsewhere in the file: additive DDL is not- # corruption recovery, so never write into a store that is not clean.- if problem := _quick_check_problem(path):+ # corruption recovery, so never write into a store that is not verified clean.+ if problem := _integrity_problem(path, budget):raise CorruptStoreError(problem)- _apply_additive_schema(path)+ _apply_additive_schema(path, budget)+ except BrowseBudgetExceeded as exc:+ raise DeclarativeDriftError(+ f"Session store for profile '{label}' is behind this Hermes version ({reason}) and could "+ f'not be verified and caught up within {BROWSE_RECONCILE_BUDGET_S:.0f}s while browsing; '+ f'nothing was changed and the list was not cleared. Retry, or start this profile\'s '+ f'gateway (`{remedy}`) or run `hermes sessions repair` to bring it up to date.') from excexcept OwnershipConflict as exc:if live_runtime_owner(home) is not None:raise owned(reason) from excdiff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.pyindex d0d25f01602..dda0c318c87 100644--- a/tests/hermes_cli/test_browse_schema_drift.py+++ b/tests/hermes_cli/test_browse_schema_drift.py@@ -506,3 +506,130 @@ def test_gateway_boot_waits_while_the_holder_has_not_published_its_record(tmp_paholder.kill()holder.wait()++_SLOW_BROWSE = r'''+import json, os, sqlite3, sys, time+from pathlib import Path+import hermes_cli.sqlite_safe_read as ssr+from fastapi import HTTPException+from hermes_cli.web_server_sessions import _open_session_db_at_path++phase = sys.argv[2]+original = ssr.connect_tracked++class SlowConnection:+ """A real sqlite connection whose VM runs slowly (stand-in for a cold multi-GB disk). The+ slowdown is installed here, whether or not the code under test sets its own progress handler,+ and chains to that handler when it does."""+ def __init__(self, conn):+ self._conn = conn+ self.set_progress_handler(None, 0)+ def __getattr__(self, name):+ return getattr(self._conn, name)+ def set_progress_handler(self, handler, n):+ def slow():+ time.sleep(0.01)+ return handler() if handler is not None else 0+ return self._conn.set_progress_handler(slow, 5)+ def close(self):+ return self._conn.close()++def holding_lock():+ """Only connections opened while THIS process holds gateway.lock are slowed."""+ lock = Path(sys.argv[1]).with_name("gateway.lock")+ try:+ return json.loads(lock.read_text()).get("pid") == os.getpid()+ except (OSError, ValueError):+ return False++def slowed(path, *a, **kw):+ conn = original(path, *a, **kw)+ read_only = "mode=ro" in str(path)+ if holding_lock() and (phase == "integrity") == read_only:+ return SlowConnection(conn)+ return conn++ssr.connect_tracked = slowed+started = time.monotonic()+print("started", flush=True)+try:+ _open_session_db_at_path(Path(sys.argv[1]), read_only=True).close()+ print("RESULT 200", time.monotonic() - started, flush=True)+except HTTPException as exc:+ print("RESULT", exc.status_code, time.monotonic() - started, exc.detail, flush=True)+'''++[email protected]("linux", "macos")[email protected]("phase", ["integrity", "ddl"])+def test_slow_browse_reconcile_gives_up_inside_its_budget_and_boot_waits_it_out(tmp_path, phase):+ """R2-2: the busy timeout bounds lock contention, not SQL execution. A slow integrity scan or+ DDL must be interrupted at the browse budget (rolled back, retryable 503), and gateway boot's+ wait must outlast that budget, so a slow browse never turns into a startup refusal."""+ import os+ import subprocess+ import sys+ import time++ import gateway.run_bootstrap as bootstrap+ import hermes_state_reconcile as reconcile+ from gateway.runtime_ownership import ProfileOwnership++ budget_s = getattr(reconcile, "BROWSE_RECONCILE_BUDGET_S", 2.0)+ wait_s = getattr(bootstrap, "_browse_reconcile_reserve_wait_s", lambda: 5.0)()+ assert wait_s > budget_s + 1+ home = _profile_home(tmp_path)+ path = _lagging_store_with_history(home, rows=400)+ before = _file_state(path)+ browse = subprocess.Popen([sys.executable, "-c", _SLOW_BROWSE, str(path), phase], env=os.environ,+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)+ assert browse.stdout.readline().strip() == "started"+ deadline = time.monotonic() + 20+ while not (home / "gateway.lock").exists() or not (home / "gateway.lock").read_text():+ assert time.monotonic() < deadline and browse.poll() is None+ time.sleep(0.01)+ booting = ProfileOwnership()+ try:+ bootstrap._reserve_waiting_out_browse_reconcile(booting, [home]) # default wait: must not raise+ assert booting.owns(home)+ finally:+ booting.close()+ out = browse.communicate(timeout=60)[0]+ result = next(line for line in out.splitlines() if line.startswith("RESULT"))+ _, status, elapsed, *detail = result.split(" ", 3)+ assert status == "503", out+ assert float(elapsed) < budget_s + 2, out+ assert "nothing was changed" in " ".join(detail)+ assert _file_state(path) == before+ assert "last_read_at" not in _session_columns(path)+++def test_equal_cardinality_corrupt_index_plus_drift_is_not_written(tmp_path):+ """R2-3: quick_check does not compare index keys with the table, so two equal-size indexes+ with swapped root pages pass it. Browse must still refuse as corrupt and write nothing."""+ path = _lagging_store_with_history(_profile_home(tmp_path), rows=10)+ with closing(sqlite3.connect(path)) as conn:+ conn.execute("CREATE TABLE probe_other (v TEXT)")+ conn.execute("CREATE INDEX probe_other_index ON probe_other(v)")+ conn.execute("INSERT INTO probe_other(v) VALUES ('wrong-source')")+ conn.commit()+ target = conn.execute("SELECT rootpage FROM sqlite_master WHERE name='idx_sessions_source'").fetchone()[0]+ other = conn.execute("SELECT rootpage FROM sqlite_master WHERE name='probe_other_index'").fetchone()[0]+ conn.execute("PRAGMA writable_schema=ON")+ conn.execute("UPDATE sqlite_master SET rootpage=? WHERE name='idx_sessions_source'", (other,))+ conn.execute("UPDATE sqlite_master SET rootpage=? WHERE name='probe_other_index'", (target,))+ conn.execute("PRAGMA writable_schema=OFF")+ conn.commit()+ conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")+ with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn:+ assert conn.execute("PRAGMA quick_check").fetchall() == [("ok",)]+ assert conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok"+ before = _file_state(path)++ with pytest.raises(HTTPException) as caught:+ _open_session_db_at_path(path, read_only=True).close()+ assert caught.value.status_code == 503+ assert caught.value.detail["error"] == "state_db_corrupt"+ assert _file_state(path) == before+ assert "last_read_at" not in _session_columns(path)+