Skip to content

The agent that grows with you

fix(sessions): catch a lagging state.db up once on browse instead of 503 (#140315)

Teknium1eda335ae044
diff --git a/hermes_cli/web_routers/profiles.py b/hermes_cli/web_routers/profiles.py
index 0884298a6f7..d55f37a86d7 100644
--- a/hermes_cli/web_routers/profiles.py
+++ b/hermes_cli/web_routers/profiles.py
@@ -274,9 +274,9 @@ def _read_profile_db(name: str, home, errors: Optional[list[dict[str, str]]],
the open fails or ``fn`` raises (warned once, recorded in ``errors`` when given).
Read-only on the healthy path: this runs on every sidebar refresh, so it must not
- routinely DDL/write-lock another profile's live DB. The open helper's stale-schema probe
- performs a ONE-TIME writable open when the store predates a schema addition — read-only
- opens skip column reconciliation and would otherwise fail here on every refresh."""
+ routinely DDL/write-lock another profile's live DB. A store that predates a schema addition
+ gets the missing declared tables/columns ONCE from the open helper, and only while no gateway
+ owns it; an owned store is reported in ``errors`` with the restart remedy (#140315)."""
db_path = Path(home) / "state.db"
if not db_path.exists():
return None
diff --git a/hermes_cli/web_server_lifecycle.py b/hermes_cli/web_server_lifecycle.py
index d3b076593f5..6a6d9f96a5b 100644
--- a/hermes_cli/web_server_lifecycle.py
+++ b/hermes_cli/web_server_lifecycle.py
@@ -177,8 +177,8 @@ def _eager_reconcile_own_session_db() -> None:
must never see a second writable ``SessionDB`` from this process (its
close-time checkpoint and a possible FTS rebuild in ``_init_fts`` are the
two-writer corruption vector, #107688 / #100896). Access-mode semantics
- (bootstrap of a missing store, ONE writable heal of a stale schema, so the
- #79531 contract holds) live in the routers' own-store opener,
+ (no bootstrap of a missing store; one additive catch-up of a store that lags
+ SCHEMA_SQL, only while no gateway owns it, #79531 / #140315) live in the routers' own-store opener,
:func:`hermes_cli.web_server_sessions._open_session_db_at_path`. Never
raises: an unfixable store still gets the per-poll read-probe heal.
"""
diff --git a/hermes_cli/web_server_sessions.py b/hermes_cli/web_server_sessions.py
index ee832df938b..5d2d82cb970 100644
--- a/hermes_cli/web_server_sessions.py
+++ b/hermes_cli/web_server_sessions.py
@@ -328,9 +328,12 @@ def _session_db_read_probe_statements() -> tuple:
def _open_session_db_at_path(db_path: Path, *, read_only: bool):
- """Browsing never initializes, repairs or upgrades the owner's session store.
+ """Browsing never initializes or repairs the owner's session store.
- Owner startup reconciles schema; explicit mutation APIs retain their writable
+ Owner startup reconciles schema. The one write a browse may do: a store that lags
+ SCHEMA_SQL (missing table/column, not corruption) with NO gateway owning it gets the
+ missing declared tables/columns once, in one short locked transaction (#140315); an
+ owned store is refused by name instead. Explicit mutation APIs retain their writable
acquisition until their separate owner-RPC migration.
"""
import sqlite3
@@ -352,7 +355,11 @@ def _open_session_db_at_path(db_path: Path, *, read_only: bool):
status_code=503,
detail="Session store is not initialized. Start this profile's gateway to initialize it.")
- try:
+ from hermes_state_reconcile import (
+ DeclarativeDriftError, is_declarative_drift_error, reconcile_drift_for_browse,
+ )
+
+ def _open_probed():
db = SessionDB(db_path=db_path, read_only=True)
try:
conn = getattr(db, "_conn", None)
@@ -363,6 +370,22 @@ def _open_session_db_at_path(db_path: Path, *, read_only: bool):
except BaseException:
db.close()
raise
+
+ try:
+ try:
+ return _open_probed()
+ except sqlite3.OperationalError as exc:
+ # A missing declared table/column is upgrade drift, not corruption (#140315): catch
+ # an unowned store up once under its runtime lock, refuse by name when a gateway
+ # owns it. Never destructive: additive DDL only, no repair from browse.
+ if not is_declarative_drift_error(exc):
+ raise
+ try:
+ reconcile_drift_for_browse(db_path)
+ except DeclarativeDriftError as drift:
+ _log.warning("session store at %s is behind SCHEMA_SQL: %s", db_path, drift)
+ raise HTTPException(status_code=503, detail=str(drift)) from exc
+ return _open_probed()
except (sqlite3.DatabaseError, UnicodeDecodeError) as exc:
from hermes_state_errors import is_malformed_db_error
if isinstance(exc, sqlite3.DatabaseError) and is_malformed_db_error(exc):
diff --git a/hermes_state_common.py b/hermes_state_common.py
index 44ca96f6d6b..343b6fc8222 100644
--- a/hermes_state_common.py
+++ b/hermes_state_common.py
@@ -281,7 +281,12 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str:
f"(SELECT started_at FROM sessions _act_s WHERE _act_s.id = {session_id_expr})")
-SCHEMA_VERSION = 31
+# v32 (#140315): no data migration. #106742 added runtime_epoch, session_admissions, worker_executions,
+# worker_receipts and three columns to SCHEMA_SQL while v31 stayed, so a store missing them read as
+# current. The bump makes that store detectably behind; its catch-up is the declarative
+# executescript(SCHEMA_SQL) + _reconcile_columns() every writable open already runs (additive,
+# idempotent). Bump again whenever SCHEMA_SQL gains a table or column.
+SCHEMA_VERSION = 32
# Auto-maintenance VACUUMs only above this freelist fraction; below it a rewrite costs more I/O than it returns.
# Auto-maintenance only VACUUMs when at least this fraction of the database file is reclaimable (``PRAGMA
diff --git a/hermes_state_reconcile.py b/hermes_state_reconcile.py
index 6dc3754367a..88c07df7591 100644
--- a/hermes_state_reconcile.py
+++ b/hermes_state_reconcile.py
@@ -1,8 +1,12 @@
-"""Explicit offline repair of additive state.db schema drift.
+"""Additive state.db schema drift: detection, the one-shot browse reconcile, and explicit repair.
-Browsing stays read-only. Reconcile a protected snapshot so failed DDL never
-leaves the user's store partly upgraded or escalates into FTS surgery.
+Drift (a declared table/column missing) is not corruption. Browsing catches an unowned store up
+once, additive DDL only (:func:`reconcile_drift_for_browse`); ``hermes sessions repair`` reconciles
+a protected snapshot so failed DDL never leaves the store partly upgraded or escalates into FTS
+surgery (:func:`repair_declarative_schema`). Neither runs while a gateway owns the store.
"""
+import contextlib
+import logging
import sqlite3
from contextlib import closing
from pathlib import Path
@@ -93,6 +97,107 @@ def _reconcile_locked(path, backup, report):
report.update(repaired=True, strategy='reconcile_declarative_schema')
+# Browse-time reconcile (#140315). A store whose tables/columns lag SCHEMA_SQL is not corrupt:
+# a writable open would catch it up, but an idle profile with no gateway never gets one, so every
+# read-only surface (sidebar, /api/sessions/<id>/messages?profile=) lost its whole history.
+# Catch it up ONCE, additive DDL only, in one short transaction, and only while holding that
+# profile's gateway runtime lock. When a gateway owns the store it is the only writer: refuse
+# by name with the remedy instead of becoming a second one.
+_BROWSE_RECONCILE_BUSY_S = 2.0
+
+
+class DeclarativeDriftError(RuntimeError):
+ """The store lags SCHEMA_SQL (missing table/column) and browsing could not catch it up."""
+
+ def __init__(self, detail: str, *, owned: bool = False):
+ super().__init__(detail)
+ self.owned = owned
+
+
+def is_declarative_drift_error(exc: BaseException) -> bool:
+ """``no such table/column`` from a schema read probe: drift, never corruption."""
+ from hermes_state_repair import _schema_not_built
+ return isinstance(exc, sqlite3.OperationalError) and _schema_not_built(exc)
+
+
+def _owner_remedy(home: Path) -> tuple[str, str]:
+ from gateway.status import _profile_label_for_home # never raises: None when unknown
+ label = _profile_label_for_home(home)
+ if label and label != 'default':
+ return label, f'hermes -p {label} gateway restart'
+ return label or str(home), 'hermes gateway restart'
+
+
+def _apply_additive_schema(path: Path) -> None:
+ """Missing SCHEMA_SQL tables and columns, in ONE ``BEGIN IMMEDIATE`` transaction.
+
+ No data migration, no FTS work, no schema_version stamp (that stays the owner's: the
+ version gates data migrations this path does not run). Every statement is additive and
+ idempotent, so a failure rolls the store back to exactly what it was."""
+ from hermes_cli.sqlite_safe_read import connect_tracked
+ from hermes_state_common import SCHEMA_SQL
+ from hermes_state_repair import _reapply_durability_barriers
+ from hermes_state_schema import SessionSchemaMixin
+
+ with closing(connect_tracked(path, tracking_path=path, timeout=_BROWSE_RECONCILE_BUSY_S,
+ isolation_level=None)) as conn:
+ _reapply_durability_barriers(conn)
+ conn.execute('BEGIN IMMEDIATE')
+ try:
+ cursor = conn.cursor()
+ shim = object.__new__(SessionSchemaMixin)
+ # Columns first: SCHEMA_SQL indexes may name columns an old table lacks.
+ shim._reconcile_columns(cursor)
+ SessionSchemaMixin._execute_ddl_script_transactional(cursor, SCHEMA_SQL)
+ shim._reconcile_columns(cursor) # tables the script just created are complete; no-op
+ conn.execute('COMMIT')
+ except BaseException:
+ with contextlib.suppress(sqlite3.Error):
+ conn.execute('ROLLBACK')
+ raise
+
+
+def reconcile_drift_for_browse(path: Path) -> bool:
+ """Catch a lagging store up for a read-only browse; True when DDL ran, False when already current.
+
+ Raises :class:`DeclarativeDriftError` (``owned=True`` when a running gateway owns the store)
+ with a user-facing detail naming the drift and the remedy. Never touches a store missing its
+ history tables: recreating them empty would hide lost sessions."""
+ from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
+
+ reason = schema_drift_reason(path)
+ if reason is None:
+ return False
+ from hermes_cli.sqlite_safe_read import connect_tracked
+ with closing(connect_tracked(path.resolve().as_uri() + '?mode=ro', uri=True, tracking_path=path)) as conn:
+ try:
+ _require_history_tables(conn)
+ except RuntimeError as exc:
+ raise DeclarativeDriftError(
+ f'Session store is missing its history tables ({reason}); this is not upgrade drift. '
+ 'Run `hermes sessions recover`.') from exc
+ label, remedy = _owner_remedy(path.parent)
+ try:
+ with exclusive_maintenance([path.parent]):
+ # Re-check under the lock: a gateway that started and released between probes
+ # (or a sibling browse) may already have caught the store up.
+ if schema_drift_reason(path) is None:
+ return False
+ _apply_additive_schema(path)
+ except OwnershipConflict as exc:
+ raise DeclarativeDriftError(
+ f"Session store for profile '{label}' is behind this Hermes version ({reason}), and a "
+ f'running gateway owns it, so browsing will not write to it. Restart that gateway '
+ f'(`{remedy}`) to bring the store up to date.', owned=True) from exc
+ if after := schema_drift_reason(path):
+ raise DeclarativeDriftError(
+ f"Session store for profile '{label}' is behind this Hermes version and could not be "
+ f'caught up ({after}). Run `hermes sessions repair` for that profile.')
+ logging.getLogger(__name__).warning(
+ 'state.db at %s lagged SCHEMA_SQL (%s); added the missing tables/columns for browsing', path, reason)
+ return True
+
+
def repair_declarative_schema(path: Path, *, backup: bool = True):
"""Reuse repair's snapshot/SQLite exclusion beneath the runtime owner lock."""
from gateway.runtime_ownership import OwnershipConflict, exclusive_maintenance
diff --git a/hermes_state_schema.py b/hermes_state_schema.py
index bf32e8ab7d9..c10288a9731 100644
--- a/hermes_state_schema.py
+++ b/hermes_state_schema.py
@@ -762,6 +762,8 @@ class SessionSchemaMixin:
rows = cursor.execute(f'PRAGMA table_info("{table_name}")').fetchall()
except sqlite3.OperationalError:
continue # Table doesn't exist yet (shouldn't happen after executescript)
+ if not rows:
+ continue # PRAGMA table_info returns [] for a missing table; the DDL creates it whole
# PRAGMA table_info rows: (cid, name, type, notnull, dflt_value, pk)
live_cols = {row[1] for row in rows}
for col_name, col_type in declared_cols.items():
@@ -1104,6 +1106,9 @@ class SessionSchemaMixin:
).fetchone() is None:
self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor)
+ # v32: nothing to run here. The declared tables/columns #106742 added under v31 were already
+ # created by _init_schema's executescript(SCHEMA_SQL) + _reconcile_columns() before this chain;
+ # the version only records that the store has them (#140315).
# Advance schema_version — deliberately NOT gated on the FTS opt-in (that would block
# every future migration for a user who never optimizes). FTS5 unavailable is the
# one skip: claiming current would lie.
diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.py
new file mode 100644
index 00000000000..37bb6b3e613
--- /dev/null
+++ b/tests/hermes_cli/test_browse_schema_drift.py
@@ -0,0 +1,260 @@
+"""A store that lags SCHEMA_SQL must stay browsable (#140315).
+
+#106742 added four tables and three columns to SCHEMA_SQL under schema_version 31. A named
+profile whose state.db had not been opened writable since the update lost its whole session
+list on every read-only surface: the browse probe hit ``no such table`` and answered 503
+"schema is unavailable or corrupt", and an idle profile never got the writable open that would
+have caught it up. These tests build the real pre-merge shape from the pre-#106742 SCHEMA_SQL
+(trimmed to what that release declared) and browse it through the real read-only path.
+"""
+from __future__ import annotations
+
+import sqlite3
+from contextlib import closing
+
+import pytest
+from fastapi import HTTPException
+
+from hermes_cli.web_server_sessions import _open_session_db_at_path
+from hermes_state import SessionDB
+from hermes_state_common import SCHEMA_VERSION
+
+_ADDED_TABLES = ("runtime_epoch", "session_admissions", "worker_executions", "worker_receipts")
+_ADDED_COLUMNS = (("sessions", "runtime_revision"), ("sessions", "runtime_generation"),
+ ("async_delegations", "owner_execution_id"))
+
+
+def _pre_merge_store(path, *, version=31):
+ """The shape a store kept by the release before #106742 has: built by SessionDB, then the
+ tables/columns #106742 added removed again, stamped with the version that release wrote."""
+ with SessionDB(db_path=path) as db:
+ db.create_session("kept", source="cli")
+ db.append_message("kept", role="user", content="history survives")
+ with closing(sqlite3.connect(path)) as conn:
+ conn.execute("DROP TRIGGER IF EXISTS sessions_runtime_ledger_delete_guard")
+ for table in reversed(_ADDED_TABLES):
+ conn.execute(f"DROP TABLE {table}")
+ conn.execute("DROP INDEX IF EXISTS idx_sessions_local_created_started")
+ for table, column in _ADDED_COLUMNS:
+ conn.execute(f"ALTER TABLE {table} DROP COLUMN {column}")
+ conn.execute("UPDATE schema_version SET version = ?", (version,))
+ conn.commit()
+ return path
+
+
+def _tables(path):
+ with closing(sqlite3.connect(path)) as conn:
+ return {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
+
+
+def _profile_home(tmp_path, name="scribe"):
+ home = tmp_path / "profiles" / name
+ home.mkdir(parents=True)
+ (home / "config.yaml").write_text("{}\n", encoding="utf-8")
+ return home
+
+
+def test_unowned_lagging_store_browses_after_one_reconcile(tmp_path):
+ path = _pre_merge_store(_profile_home(tmp_path) / "state.db")
+
+ db = _open_session_db_at_path(path, read_only=True)
+ try:
+ assert [row["id"] for row in db.list_sessions_rich(limit=10, compact_rows=True)] == ["kept"]
+ assert [m["content"] for m in db.get_messages("kept")] == ["history survives"]
+ finally:
+ db.close()
+ assert set(_ADDED_TABLES) <= _tables(path)
+
+ # Once: the second browse is a pure read-only open (no write, no new DDL).
+ before = path.read_bytes()
+ db = _open_session_db_at_path(path, read_only=True)
+ db.close()
+ assert path.read_bytes() == before
+
+
+def test_owned_lagging_store_is_not_written_and_names_the_remedy(tmp_path):
+ from gateway.runtime_ownership import ProfileOwnership
+
+ home = _profile_home(tmp_path)
+ path = _pre_merge_store(home / "state.db")
+ before = path.read_bytes()
+ owner = ProfileOwnership() # what a running gateway holds on its profile
+ owner.reserve([home])
+ try:
+ with pytest.raises(HTTPException) as caught:
+ _open_session_db_at_path(path, read_only=True)
+ finally:
+ owner.close()
+ assert caught.value.status_code == 503
+ detail = str(caught.value.detail)
+ assert "running gateway owns it" in detail
+ assert "gateway restart" in detail
+ assert "corrupt" not in detail
+ assert path.read_bytes() == before
+ assert not set(_ADDED_TABLES) & _tables(path)
+
+
+def test_corrupt_store_still_returns_503(tmp_path):
+ """Corruption is not drift: the open fails with the corrupt-store 503 and nothing is written."""
+ path = tmp_path / "state.db"
+ with SessionDB(db_path=path) as db:
+ db.create_session("kept", source="cli")
+ with closing(sqlite3.connect(path)) as conn:
+ if conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal":
+ conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+ page_size = conn.execute("PRAGMA page_size").fetchone()[0]
+ with open(path, "r+b") as fh: # sqlite_master (page 1 b-tree) overwritten past the header
+ fh.seek(100)
+ fh.write(b"\xde\xad\xbe\xef" * ((page_size - 100) // 4))
+ before = path.read_bytes()
+
+ with pytest.raises(HTTPException) as caught:
+ _open_session_db_at_path(path, read_only=True).close()
+ assert caught.value.status_code == 503
+ assert caught.value.detail["error"] == "state_db_corrupt"
+ assert path.read_bytes() == before
+
+
+def test_missing_history_table_is_refused_not_recreated(tmp_path):
+ path = _pre_merge_store(tmp_path / "state.db")
+ with closing(sqlite3.connect(path)) as conn:
+ conn.execute("PRAGMA foreign_keys=OFF")
+ conn.execute("DROP TABLE messages")
+ conn.commit()
+ before = path.read_bytes()
+ with pytest.raises(HTTPException) as caught:
+ _open_session_db_at_path(path, read_only=True)
+ assert caught.value.status_code == 503
+ assert "sessions recover" in str(caught.value.detail)
+ assert path.read_bytes() == before
+
+
+def test_version_bump_migration_is_additive_and_idempotent(tmp_path):
+ path = _pre_merge_store(tmp_path / "state.db")
+ assert SCHEMA_VERSION > 31
+
+ for _ in range(2):
+ with SessionDB(db_path=path) as db:
+ assert db._conn.execute("SELECT version FROM schema_version").fetchone()[0] == SCHEMA_VERSION
+ assert db._conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0] == 1
+ assert set(_ADDED_TABLES) <= _tables(path)
+ with closing(sqlite3.connect(path)) as conn:
+ for table, column in _ADDED_COLUMNS:
+ assert column in {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
+
+ # A browse-reconciled store (still stamped 31) also upgrades cleanly on the owner's open.
+ other = _pre_merge_store(tmp_path / "other.db")
+ _open_session_db_at_path(other, read_only=True).close()
+ with SessionDB(db_path=other) as db:
+ assert db._conn.execute("SELECT version FROM schema_version").fetchone()[0] == SCHEMA_VERSION
+
+
+def test_messages_route_for_lagging_profile(monkeypatch, tmp_path):
+ """``GET /api/sessions/<id>/messages?profile=<name>`` was the 503 in the report."""
+ try:
+ from starlette.testclient import TestClient
+ except ImportError:
+ pytest.skip("fastapi/starlette not installed")
+ from hermes_cli import profiles as profiles_mod
+ from hermes_cli.web_server import _SESSION_HEADER_NAME, _SESSION_TOKEN, app
+
+ home = _profile_home(tmp_path)
+ _pre_merge_store(home / "state.db")
+ monkeypatch.setattr(profiles_mod, "profile_exists", lambda name: name == "scribe")
+ monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda name: home)
+ client = TestClient(app)
+ client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN
+
+ response = client.get("/api/sessions/kept/messages?profile=scribe")
+ assert response.status_code == 200, response.text
+ assert [m["content"] for m in response.json()["messages"]] == ["history survives"]
+
+
+# ── Desktop/dashboard HTTP surfaces on this process's own store (moved from test_web_server.py) ──
+
+
+def own_store_client(monkeypatch, _isolate_hermes_home):
+ try:
+ from starlette.testclient import TestClient
+ except ImportError:
+ pytest.skip("fastapi/starlette not installed")
+ import hermes_state
+ from hermes_cli.web_routers import profiles as profiles_routes
+ from hermes_cli.web_server import _SESSION_HEADER_NAME, _SESSION_TOKEN, app
+ from hermes_constants import get_hermes_home
+
+ monkeypatch.setattr(profiles_routes, "_SIDEBAR_CACHE_TTL_SECONDS", 0.0)
+ path = get_hermes_home() / "state.db"
+ monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", path)
+ (get_hermes_home() / "config.yaml").write_text("{}\n", encoding="utf-8")
+ with SessionDB(db_path=path) as db:
+ db.create_session("stale-schema", source="cli")
+ db.append_message("stale-schema", role="user", content="hi")
+ client = TestClient(app)
+ client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN
+ return client, path
+
+
+def _drop_session_column(path, column):
+ with closing(sqlite3.connect(path)) as conn:
+ # SQLite refuses DROP COLUMN while an index references it; a pre-column store has neither.
+ conn.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity")
+ conn.execute(f"ALTER TABLE sessions DROP COLUMN {column}")
+ conn.commit()
+
+
+def _session_columns(path):
+ with closing(sqlite3.connect(path)) as conn:
+ return {row[1] for row in conn.execute("PRAGMA table_info(sessions)")}
+
+
+def _owned(home):
+ from gateway.runtime_ownership import ProfileOwnership
+ owner = ProfileOwnership() # what a running gateway holds on its profile
+ owner.reserve([home])
+ return owner
+
+
[email protected]("missing_column", ["archived", "pinned", "last_activity_at"])
+def test_session_list_reconciles_once_only_when_unowned(own_store_client, missing_column):
+ client, path = own_store_client
+ _drop_session_column(path, missing_column)
+
+ before = path.read_bytes()
+ owner = _owned(path.parent)
+ try:
+ response = client.get("/api/sessions?limit=50&offset=0")
+ sidebar = client.get("/api/profiles/sessions/sidebar").json()
+ finally:
+ owner.close()
+ assert response.status_code == 503
+ assert "gateway owns it" in response.json()["detail"]
+ # A failed read is a retryable failed load naming the drift, never an empty list.
+ assert "behind this Hermes version" in sidebar["errors"][0]["error"]
+ assert "sessions" not in sidebar["recents"] and sidebar["recents"]["failed"] is True
+ assert path.read_bytes() == before
+ assert missing_column not in _session_columns(path)
+
+ response = client.get("/api/sessions?limit=50&offset=0")
+ assert response.status_code == 200
+ assert [row["id"] for row in response.json()["sessions"]] == ["stale-schema"]
+ assert missing_column in _session_columns(path)
+
+
+def test_startup_eager_reconcile_heals_only_an_unowned_stale_store(own_store_client):
+ """After `hermes update` an old-schema store must not fail every poll (#79531/#80037); the
+ dashboard is never a second writer while a gateway owns it (#107688)."""
+ from hermes_cli import web_server_lifecycle
+
+ _client, path = own_store_client
+ _drop_session_column(path, "last_read_at")
+ owner = _owned(path.parent)
+ try:
+ web_server_lifecycle._eager_reconcile_own_session_db()
+ finally:
+ owner.close()
+ assert "last_read_at" not in _session_columns(path)
+
+ web_server_lifecycle._eager_reconcile_own_session_db()
+ assert "last_read_at" in _session_columns(path)
diff --git a/tests/hermes_cli/test_sessions_repair_declarative.py b/tests/hermes_cli/test_sessions_repair_declarative.py
index 88ddcd5c01d..6c402d45280 100644
--- a/tests/hermes_cli/test_sessions_repair_declarative.py
+++ b/tests/hermes_cli/test_sessions_repair_declarative.py
@@ -9,7 +9,13 @@ from hermes_state import SessionDB
from hermes_state_common import SCHEMA_VERSION
from hermes_cli import sessions_cmd
from hermes_cli.web_server_sessions import _open_session_db_at_path
-from fastapi import HTTPException
+
+
[email protected](autouse=True)
+def _volume_headroom(monkeypatch):
+ """The backup/snapshot guards reserve 2% of the volume: hermetic on a nearly full dev disk."""
+ import hermes_state_repair as repair
+ monkeypatch.setattr(repair, '_REPAIR_BACKUP_FREE_FRACTION', 0.0)
def _legacy_store(tmp_path, monkeypatch):
@@ -33,9 +39,7 @@ def _legacy_store(tmp_path, monkeypatch):
def test_check_only_reports_drift_without_mutation(tmp_path, monkeypatch):
path = _legacy_store(tmp_path, monkeypatch)
before = path.read_bytes()
- with pytest.raises(HTTPException) as exc:
- _open_session_db_at_path(path, read_only=True)
- assert exc.value.status_code == 503
+ # Browsing would catch this unowned store up (#140315); check-only must not.
assert sessions_cmd._cmd_repair(SimpleNamespace(check_only=True)) == 1
assert path.read_bytes() == before
diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py
index ecaa37ceb97..ed942172b6e 100644
--- a/tests/hermes_cli/test_web_server.py
+++ b/tests/hermes_cli/test_web_server.py
@@ -418,144 +418,6 @@ class TestWebServerEndpoints:
assert "not initialized" in response.json()["detail"]
assert set(get_hermes_home().glob("state.db*")) == before
- @pytest.mark.parametrize(
- "missing_column", ["archived", "pinned", "last_activity_at"]
- )
- def test_get_sessions_reports_stale_schema_without_healing(self, missing_column):
- import sqlite3
-
- from hermes_constants import get_hermes_home
- from hermes_state import SessionDB
-
- db_path = get_hermes_home() / "state.db"
- seed = SessionDB(db_path=db_path)
- try:
- seed.create_session("stale-schema", source="cli")
- finally:
- seed.close()
-
- legacy = sqlite3.connect(str(db_path))
- try:
- # SQLite refuses DROP COLUMN while an index references the
- # column; a pre-column legacy store has neither.
- legacy.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity")
- legacy.execute(f"ALTER TABLE sessions DROP COLUMN {missing_column}")
- legacy.commit()
- finally:
- legacy.close()
-
- before = db_path.read_bytes()
- response = self.client.get("/api/sessions?limit=50&offset=0")
-
- assert response.status_code == 503
- assert "schema" in response.json()["detail"]
- assert db_path.read_bytes() == before
- healed = sqlite3.connect(str(db_path))
- try:
- columns = {
- row[1] for row in healed.execute("PRAGMA table_info(sessions)")
- }
- finally:
- healed.close()
- assert missing_column not in columns
-
- def test_profiles_sidebar_reports_stale_schema_store(self):
- """Aggregation reports unavailable profiles instead of healing their DBs."""
- import sqlite3
-
- from hermes_constants import get_hermes_home
- from hermes_state import SessionDB
-
- db_path = get_hermes_home() / "state.db"
- seed = SessionDB(db_path=db_path)
- try:
- seed.create_session("sidebar-stale", source="cli")
- seed.append_message(
- session_id="sidebar-stale", role="user", content="hi"
- )
- finally:
- seed.close()
-
- legacy = sqlite3.connect(str(db_path))
- try:
- legacy.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity")
- legacy.execute("ALTER TABLE sessions DROP COLUMN last_activity_at")
- legacy.commit()
- finally:
- legacy.close()
-
- response = self.client.get("/api/profiles/sessions/sidebar")
-
- assert response.status_code == 200
- payload = response.json()
- assert payload["errors"]
- assert "schema" in payload["errors"][0]["error"]
- # A failed read is a retryable failed load, never an empty list.
- assert "sessions" not in payload["recents"]
- assert payload["recents"]["failed"] is True
-
- def test_startup_eager_reconcile_heals_stale_store(self):
- """The OWNER's startup open brings a stale store current; the dashboard only reads.
-
- #79531/#80037: after `hermes update` an old-schema state.db used to
- stay stale until the first NEW session forced a writable open —
- every /api/sessions poll 500ed with "no such column" in between.
- Under the unified runtime the dashboard is a zero-writer view: its
- eager probe must NOT heal (#107688 two-writer vector), and the
- gateway owner's writable acquisition is what reconciles the schema.
- """
- import sqlite3
-
- from hermes_constants import get_hermes_home
- from hermes_state import SessionDB
-
- db_path = get_hermes_home() / "state.db"
- seed = SessionDB(db_path=db_path)
- try:
- seed.create_session("eager-stale", source="cli")
- finally:
- seed.close()
-
- legacy = sqlite3.connect(str(db_path))
- try:
- legacy.execute("ALTER TABLE sessions DROP COLUMN last_read_at")
- legacy.commit()
- finally:
- legacy.close()
-
- # Dashboard probe: read-only, never heals (no writable open from a viewer).
- _web_server_lifecycle._eager_reconcile_own_session_db()
- stale = sqlite3.connect(str(db_path))
- try:
- assert "last_read_at" not in {row[1] for row in stale.execute("PRAGMA table_info(sessions)")}
- finally:
- stale.close()
-
- # Owner path: the gateway's writable acquisition reconciles the schema at startup.
- from hermes_state_registry import acquire, release_or_close
- owner = acquire(db_path)
- try:
- pass
- finally:
- release_or_close(owner)
-
- healed = sqlite3.connect(str(db_path))
- try:
- columns = {
- row[1] for row in healed.execute("PRAGMA table_info(sessions)")
- }
- finally:
- healed.close()
- assert "last_read_at" in columns
-
- # The healed store serves the full rich listing.
- db = SessionDB(db_path=db_path, read_only=True)
- try:
- rows = db.list_sessions_rich(limit=10, compact_rows=True)
- finally:
- db.close()
- assert [r["id"] for r in rows] == ["eager-stale"]
-
def test_startup_eager_reconcile_is_read_only_on_a_healthy_store(self, monkeypatch):
"""A current-schema store gets NO writable open from the dashboard (#107688).