The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(state): one structural FTS layout-drift answer for the stamp, doctor and the update notice
yoniebansf42c307ef871
diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.pyindex 1e2ab5b2c24..e6221d01368 100644--- a/hermes_cli/update_cmd_maint.py+++ b/hermes_cli/update_cmd_maint.py@@ -134,15 +134,14 @@ def _print_fts_optimize_available_notice() -> None:if size_gb < 0.5:returndb = None- needs_upgrade = False+ legacy_inline = False+ drift: list[str] = []try:db = SessionDB(db_path=db_path, read_only=True)# read_only opens skip schema init; probe the stored layout directly.- row = db._conn.execute(- "SELECT sql FROM sqlite_master "- "WHERE type = 'table' AND name = 'messages_fts'"- ).fetchone()- needs_upgrade = bool(row) and db._db_needs_fts_storage_upgrade(db._conn)+ from hermes_state_fts import fts_layout_drift+ drift = fts_layout_drift(db._conn)+ legacy_inline = db._db_has_legacy_inline_fts(db._conn)# Interrupted optimize-storage: v23 table shape but backfill markers / trash# tables remain. Re-running resumes it, so offer the command again.interrupted = bool(@@ -165,7 +164,7 @@ def _print_fts_optimize_available_notice() -> None:if db is not None:with suppress(Exception):db.close()- if not needs_upgrade and not interrupted:+ if not drift and not interrupted:return # current layout already present (fresh/optimized)if interrupted:@@ -179,22 +178,31 @@ def _print_fts_optimize_available_notice() -> None:print(" hermes sessions optimize-storage")return- est_reclaim = size_gb * 0.6print()- if mode == "require":- print("◆ Session database upgrade required")- print(- f" Your search index uses the OLD storage layout and should be "- f"upgraded. The new layout typically frees ~60% of state.db "- f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is "- f"required for continued optimal operation."- )+ if legacy_inline:+ # The ~60% figure was measured for this transition only (inline copies of every message dropped).+ est_reclaim = size_gb * 0.6+ if mode == "require":+ print("◆ Session database upgrade required")+ print(+ f" Your search index uses the OLD storage layout and should be "+ f"upgraded. The new layout typically frees ~60% of state.db "+ f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is "+ f"required for continued optimal operation."+ )+ else:+ print("◆ Reclaim ~60% of your session database disk")+ print(+ f" Your search index uses the old storage layout. Upgrading it "+ f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB "+ f"of your current {size_gb:.1f} GB."+ )else:- print("◆ Reclaim ~60% of your session database disk")+ print("◆ Search index update available")print(- f" Your search index uses the old storage layout. Upgrading it "- f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB "- f"of your current {size_gb:.1f} GB."+ f" Your search index is on the compact layout from an earlier "+ f"optimization, but a later release changed it. This is a smaller "+ f"follow-up rebuild than the first optimization.")print(" Run when convenient: hermes sessions optimize-storage")print(diff --git a/hermes_state_dbfile.py b/hermes_state_dbfile.pyindex 483683ffb85..a5ae94188f5 100644--- a/hermes_state_dbfile.py+++ b/hermes_state_dbfile.py@@ -801,8 +801,8 @@ def collect_state_db_stats(db_path: Path) -> dict[str, Any]:# Raw state_meta reads — cheap, and independent of SessionDB.stats["fts_storage_version"] = _meta_int("fts_storage_version")with contextlib.suppress(Exception):- from hermes_state_fts import SessionFtsSetupMixin- stats["fts_storage_upgrade_needed"] = SessionFtsSetupMixin._db_needs_fts_storage_upgrade(conn.cursor())+ from hermes_state_fts import fts_layout_drift+ stats["fts_storage_upgrade_needed"] = bool(fts_layout_drift(conn.cursor()))stats["fts_rebuild_high_water"] = high_water = _meta_int("fts_rebuild_high_water")stats["fts_rebuild_progress"] = progress = _meta_int("fts_rebuild_progress")stats["fts_rebuild_pending"] = False if high_water is None else (progress or 0) < high_waterdiff --git a/hermes_state_fts.py b/hermes_state_fts.pyindex 3ced10cb2c2..f4d0531ae17 100644--- a/hermes_state_fts.py+++ b/hermes_state_fts.py@@ -10,8 +10,8 @@ from pathlib import Pathfrom typing import Sequencefrom hermes_constants import get_hermes_home-from hermes_state_common import (FTS_CJK_STALE_KEY, FTS_STALE_KEY, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS,- routed_sessions_setting)+from hermes_state_common import (FTS_CJK_STALE_KEY, FTS_SQL, FTS_STALE_KEY, FTS_TRIGRAM_SQL, _FTS_CJK_TRIGGERS,+ _FTS_TRIGGERS, routed_sessions_setting)from hermes_state_errors import is_fts_scoped_corruption_error, is_sqlite_lock_error# caplog tests pin the "hermes_state" logger name.@@ -127,6 +127,37 @@ def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:return False+_FTS_DDL_TABLES = ("messages_fts", "messages_fts_trigram")+++def _expected_vtable_sql(ddl: str, table: str) -> str:+ """The stored-form ``CREATE VIRTUAL TABLE`` for *table* that *ddl* would create (``IF NOT EXISTS`` and+ ``;`` dropped, whitespace collapsed), i.e. what ``sqlite_master.sql`` holds once the DDL has run."""+ marker = f"CREATE VIRTUAL TABLE IF NOT EXISTS {table} USING"+ start = ddl.index(marker)+ end = ddl.index(";", start)+ return " ".join(ddl[start:end].replace(" IF NOT EXISTS", "", 1).split())+++def fts_layout_drift(cursor: sqlite3.Cursor) -> list[str]:+ """FTS tables whose stored definition differs from what this code creates; empty when current.++ The one structural answer to "does optimize-storage have layout work": the update notice, doctor, the+ open-time stamp and optimize-storage all read it, so a layout bump needs no new per-version predicate+ and the ``fts_storage_version`` stamp is only ever written over tables that match it. An absent table+ is not drift (the trigram index is optional; the base index is created on open). The cjk index is+ left out: it exists only where its tokenizer loads and owns its own stale/backfill markers.+ """+ expected = {"messages_fts": _expected_vtable_sql(FTS_SQL, "messages_fts"),+ "messages_fts_trigram": _expected_vtable_sql(FTS_TRIGRAM_SQL, "messages_fts_trigram")}+ drift = []+ for table in _FTS_DDL_TABLES:+ row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", (table,)).fetchone()+ if row is not None and " ".join((row[0] or "").split()) != expected[table]:+ drift.append(table)+ return drift++# FTS5 shadow tables the virtual-table engine owns. `sqlite3 .recover` re-emits them# as ordinary tables but cannot re-emit the CREATE VIRTUAL TABLE row, so every later# CREATE VIRTUAL TABLE fails with "fts5: error creating shadow table <name>: table@@ -210,8 +241,8 @@ class SessionFtsSetupMixin:@classmethoddef _db_needs_fts_storage_upgrade(cls, cursor: sqlite3.Cursor) -> bool:- """True when the current FTS storage layout should be treated as stale (optimize-storage has work)."""- return cls._db_has_legacy_inline_fts(cursor) or cls._db_has_trigram_tool_calls_projection(cursor)+ """True when optimize-storage has layout work: a stored FTS vtable differs from what this code creates."""+ return bool(fts_layout_drift(cursor))def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None:"""Log once that the trigram tokenizer is missing; base FTS5 stays enabled."""diff --git a/hermes_state_schema.py b/hermes_state_schema.pyindex edde611e980..289c7573178 100644--- a/hermes_state_schema.py+++ b/hermes_state_schema.py@@ -26,7 +26,7 @@ from hermes_state_common import (LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,)-from hermes_state_fts import _drop_orphan_fts_shadow_tables+from hermes_state_fts import _drop_orphan_fts_shadow_tables, fts_layout_driftfrom hermes_state_holders import _read_proc_argvfrom hermes_state_search import _delete_meta, _meta_rowfrom hermes_state_errors import is_sqlite_lock_error@@ -371,7 +371,6 @@ class SessionSchemaMixin:cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")cursor.execute(_CLEAR_REBUILD_MARKERS_SQL)cursor.execute(_DROP_RETIRED_TOOL_HIGH_WATER_SQL)- cursor.execute(_STATE_META_UPSERT_SQL, ("fts_storage_version", str(FTS_STORAGE_VERSION)))if not has_messages:# Nothing indexed and nothing to index: swap the shape in place, no rebuild authority needed.@@ -1004,8 +1003,32 @@ class SessionSchemaMixin:self._ensure_unique_title_index(cursor)if fts5_available:self._init_fts(cursor)+ self._stamp_settled_fts_layout(cursor)self._conn.commit()+ def _stamp_settled_fts_layout(self, cursor: sqlite3.Cursor) -> None:+ """Write ``fts_storage_version`` once the stored layout is current and nothing is in flight.++ The only open-time writer of the stamp, run after _init_fts so a fresh store is stamped on creation+ and a realigned base index is stamped only when the trigram also matches (``fts_layout_drift``). An+ interrupted optimize (markers, trash, an empty external index against non-empty messages) keeps the+ stamp unwritten so the resume offer stays alive. Reads before writing: a settled store never takes+ the write lock here.+ """+ if (+ self._fts_stale+ or fts_layout_drift(cursor)+ or _meta_row(cursor, "fts_rebuild_high_water") is not None+ or self._has_fts_trash(cursor)+ or self._fts_external_index_empty_with_messages(cursor)+ ):+ return+ if cursor.execute(+ "SELECT 1 FROM state_meta WHERE key = 'fts_storage_version' AND value = ? LIMIT 1",+ (str(FTS_STORAGE_VERSION),),+ ).fetchone() is None:+ cursor.execute(_STATE_META_UPSERT_SQL, ("fts_storage_version", str(FTS_STORAGE_VERSION)))+def _run_data_migrations(self, cursor: sqlite3.Cursor, current_version: int, fts5_available: bool) -> None:"""Version-gated chain for DATA migrations only (row backfills); column additions neverbelong here. Advances schema_version at the end unless FTS5 is unavailable."""@@ -1069,10 +1092,7 @@ class SessionSchemaMixin:# Rebuild once so rows indexed by older view/trigger definitions do not linger.fts_migrations_complete = self._migrate_trigram_cron_exclusion(cursor)- # Stamp the FTS layout version (fresh/optimized DBs); a legacy DB keeps its absent/0- # marker until optimize-storage runs. An INTERRUPTED optimize (markers, trash, or an- # empty external index against non-empty messages) is NOT stamped: the marker is the- # source of truth for "fully optimized" and keeps the resume offer alive.+ # The FTS layout stamp is written by _stamp_settled_fts_layout after _init_fts, not here.# v23: FTS storage redesign (issues #22478, #43690, #55233). The v11 inline-mode FTS tables each# store a full private copy of every message (content || tool_name || tool_calls), and the trigram# index additionally covers role='tool' rows (~90% of message bytes: base64 payloads, file dumps) at@@ -1089,23 +1109,6 @@ class SessionSchemaMixin:# marker (see _fts_storage_version / SETTLE below), so schema_version advances to SCHEMA_VERSION# here like every other migration — future v24+ migrations land automatically for legacy-FTS users# too. Only the FTS *layout* waits for opt-in.- if (- fts5_available- and not self._db_needs_fts_storage_upgrade(cursor)- and cursor.execute(- "SELECT 1 FROM state_meta WHERE key = 'fts_rebuild_high_water' LIMIT 1"- ).fetchone() is None- and not self._has_fts_trash(cursor)- and not self._fts_external_index_empty_with_messages(cursor)- ):- # Stamp only when it would change something: on a settled DB every condition above- # already holds, and re-writing the same value takes the write lock on every open.- if cursor.execute(- "SELECT 1 FROM state_meta WHERE key = 'fts_storage_version' AND value = ? LIMIT 1",- (str(FTS_STORAGE_VERSION),),- ).fetchone() is None:- self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor)-# v32: nothing to run here. The declared tables/columns #106742 added under v31 were already# created by _init_schema's executescript(SCHEMA_SQL) + _reconcile_columns() before this chain;# the version only records that the store has them (#140315).diff --git a/hermes_state_search.py b/hermes_state_search.pyindex 95caec48254..c4b8ea84cfc 100644--- a/hermes_state_search.py+++ b/hermes_state_search.py@@ -19,6 +19,7 @@ from hermes_state_common import (MAX_FTS5_QUERY_CHARS, SCHEMA_VERSION, _FTS_CJK_TRIGGERS,escape_like as _escape_like, fts_rebuild_admission, fts_trigram_session_sql, routed_sessions_setting,)+from hermes_state_fts import fts_layout_drift# Pre-split logger identity so log filtering/capture is unchanged.logger = logging.getLogger("hermes_state")@@ -643,6 +644,8 @@ class SessionSearchMixin:return "teardown_incomplete"if self._fts_external_index_empty_with_messages(conn):return "backfill_incomplete"+ if fts_layout_drift(conn):+ return "layout_drift"self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn)_delete_meta(conn, "fts_optimize_available")conn.execute("UPDATE schema_version SET version = ? WHERE version < ?", (SCHEMA_VERSION, SCHEMA_VERSION))diff --git a/tests/hermes_cli/test_fts_optimize_notice.py b/tests/hermes_cli/test_fts_optimize_notice.pyindex a8b8ea36c27..01ce33348ff 100644--- a/tests/hermes_cli/test_fts_optimize_notice.py+++ b/tests/hermes_cli/test_fts_optimize_notice.py@@ -33,6 +33,7 @@ def test_update_notice_offers_v1_trigram_tool_calls_rebuild(tmp_path, monkeypatc_db_needs_fts_storage_upgrade = staticmethod(hermes_state.SessionDB._db_needs_fts_storage_upgrade)+ _db_has_legacy_inline_fts = staticmethod(hermes_state.SessionDB._db_has_legacy_inline_fts)monkeypatch.setattr(hermes_constants, "get_hermes_home", lambda: tmp_path)monkeypatch.setattr(hermes_state, "SessionDB", FakeSessionDB)@@ -64,6 +65,8 @@ def test_doctor_collects_structural_fts_upgrade_status("""Real read-only collection must ignore forged or lagging version markers."""from hermes_cli import doctor_statefrom hermes_state_dbfile import collect_state_db_stats+ from hermes_state_fts import _expected_vtable_sql+ from hermes_state_common import FTS_SQL, FTS_TRIGRAM_SQLdb_path = tmp_path / "state.db"with sqlite3.connect(db_path) as conn:@@ -75,13 +78,11 @@ def test_doctor_collects_structural_fts_upgrade_status(if base_layout == "legacy":conn.execute("CREATE VIRTUAL TABLE messages_fts USING fts5(content)")elif base_layout == "current":- conn.execute(- "CREATE VIRTUAL TABLE messages_fts USING "- "fts5(content, tool_name, tool_calls, content='messages')"- )- if trigram_layout != "absent":- columns = "content, tool_name" + (", tool_calls" if trigram_layout == "legacy" else "")- conn.execute(f"CREATE VIRTUAL TABLE messages_fts_trigram USING fts5({columns}, tokenize='trigram')")+ conn.execute(_expected_vtable_sql(FTS_SQL, "messages_fts"))+ if trigram_layout == "legacy":+ conn.execute("CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content, tool_name, tool_calls, tokenize='trigram')")+ elif trigram_layout == "current":+ conn.execute(_expected_vtable_sql(FTS_TRIGRAM_SQL, "messages_fts_trigram"))if marker is not None:conn.execute("INSERT INTO state_meta VALUES ('fts_storage_version', ?)", (str(marker),))@@ -104,13 +105,15 @@ def test_doctor_collects_structural_fts_upgrade_status(def test_doctor_collects_pending_rebuild_with_settled_layout(tmp_path, monkeypatch, progress, pending):from hermes_cli import doctor_statefrom hermes_state_dbfile import collect_state_db_stats+ from hermes_state_fts import _expected_vtable_sql+ from hermes_state_common import FTS_SQL, FTS_TRIGRAM_SQLdb_path = tmp_path / "state.db"with sqlite3.connect(db_path) as conn:conn.executescript("CREATE TABLE state_meta (key TEXT PRIMARY KEY, value TEXT);"- "CREATE VIRTUAL TABLE messages_fts USING fts5(content, tool_name, tool_calls);"- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content, tool_name, tokenize='trigram');"+ f"{_expected_vtable_sql(FTS_SQL, 'messages_fts')};"+ f"{_expected_vtable_sql(FTS_TRIGRAM_SQL, 'messages_fts_trigram')};""INSERT INTO state_meta VALUES ('fts_rebuild_high_water', '10');")conn.execute("INSERT INTO state_meta VALUES ('fts_rebuild_progress', ?)", (str(progress),))diff --git a/tests/hermes_state/test_hermes_state.py b/tests/hermes_state/test_hermes_state.pyindex 4946afb4f6f..61542351520 100644--- a/tests/hermes_state/test_hermes_state.py+++ b/tests/hermes_state/test_hermes_state.py@@ -4705,6 +4705,9 @@ class TestFTSExternalContentMigration:try:db.create_session(session_id="s1", source="cli")db.append_message("s1", role="user", content="settle guard needle")+ # A fresh store is stamped on creation; clear it so the assertion reads settle's own write.+ db._conn.execute("DELETE FROM state_meta WHERE key = 'fts_storage_version'")+ db._conn.commit()# Plant markers without going through demote.db.set_meta("fts_rebuild_high_water", "1")db.set_meta("fts_rebuild_progress", "0")