The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(state): recognise old FTS layouts by name; stamp only the exact current one
yoniebans281d5cead05d
diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.pyindex b9ec8522d41..fca53626866 100644--- a/hermes_cli/config_defaults.py+++ b/hermes_cli/config_defaults.py@@ -2310,8 +2310,8 @@ DEFAULT_CONFIG = {# Notice about the compact FTS layout (reclaims ~60%+ of state.db). OPT-IN: legacy indexes# stay until `hermes sessions optimize-storage` runs, since the rebuild is disk-heavy on# large DBs. advise = `hermes update` prints a one-line notice with reclaimable size when a- # legacy index is detected; require = shown as a REQUIRED upgrade (tooling may gate on it);- # off = none.+ # legacy index is detected (a later layout change gets a rebuild notice without a size estimate);+ # require = the same notices worded as REQUIRED (tooling may gate on it); off = none."fts_optimize_notice": "advise",# CJK-bigram search index (messages_fts_cjk). When the extension is built# (native/fts5_cjk/build.sh → ~/.hermes/lib/libfts5_cjk.so), 1-2 char CJK terms get exactdiff --git a/hermes_cli/doctor_state.py b/hermes_cli/doctor_state.pyindex c796add0c79..b74ce1399b9 100644--- a/hermes_cli/doctor_state.py+++ b/hermes_cli/doctor_state.py@@ -93,11 +93,12 @@ def _render_state_db_stats(stats: dict, holders=None, host_note: str = "") -> li"(stop the listed processes; the host gateway's own retry then rebuilds, or run ""'hermes sessions optimize-storage' with every holder stopped)"))# Oversized DB: suggest auto_prune, plus the offline optimize-storage pass when the FTS rebuild is- # pending OR the stored FTS layout structurally needs upgrading (not just a lagging marker).+ # pending, an earlier run stopped before tearing down the old index, or the stored FTS layout is an+ # older release's (not just a lagging marker).if logical is not None and logical > STATE_DB_SIZE_WARN_BYTES:detail = "consider enabling sessions.auto_prune in config.yaml to bound growth"structural_fts_upgrade = bool(stats.get("fts_storage_upgrade_needed"))- if stats.get("fts_rebuild_pending") or structural_fts_upgrade:+ if stats.get("fts_rebuild_pending") or stats.get("fts_teardown_pending") or structural_fts_upgrade:detail += "; run 'hermes sessions optimize-storage' offline (with the host gateway stopped) to compact FTS storage"lines.append(("warn", f"state.db is large ({_human_bytes(logical)})", f"({detail})"))# WAL runaway is deliberately NOT warned here: _state_db_wal already warns above 50 MB and offers --fix.diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.pyindex e6221d01368..079a99591ba 100644--- a/hermes_cli/update_cmd_maint.py+++ b/hermes_cli/update_cmd_maint.py@@ -198,11 +198,14 @@ def _print_fts_optimize_available_notice() -> None:f"of your current {size_gb:.1f} GB.")else:- print("◆ Search index update available")+ if mode == "require":+ print("◆ Search index rebuild required by your notice policy")+ else:+ print("◆ Search index update available")print(- f" Your search index is on the compact layout from an earlier "- f"optimization, but a later release changed it. This is a smaller "- f"follow-up rebuild than the first optimization."+ " Your search index is on the compact layout from an earlier "+ "optimization, but a later release changed it. Running the "+ "command again rebuilds it in the current layout.")print(" Run when convenient: hermes sessions optimize-storage")print(diff --git a/hermes_state_dbfile.py b/hermes_state_dbfile.pyindex a5ae94188f5..a6dff2d948e 100644--- a/hermes_state_dbfile.py+++ b/hermes_state_dbfile.py@@ -752,7 +752,8 @@ def collect_state_db_stats(db_path: Path) -> dict[str, Any]:from hermes_state import _connect_tracked_dbstats: dict[str, Any] = dict.fromkeys(("page_count", "page_size", "freelist_count", "logical_size_bytes", "wal_size_bytes", "journal_mode",- "messages", "sessions", "fts_tables", "fts_storage_version", "fts_storage_upgrade_needed", "fts_rebuild_pending",+ "messages", "sessions", "fts_tables", "fts_storage_version", "fts_storage_upgrade_needed",+ "fts_teardown_pending", "fts_rebuild_pending","fts_rebuild_high_water", "fts_rebuild_progress", "fts_rebuild_deferral"))# WAL sidecar size needs no connection at all.with contextlib.suppress(OSError):@@ -803,6 +804,11 @@ def collect_state_db_stats(db_path: Path) -> dict[str, Any]:with contextlib.suppress(Exception):from hermes_state_fts import fts_layout_driftstats["fts_storage_upgrade_needed"] = bool(fts_layout_drift(conn.cursor()))+ with contextlib.suppress(Exception):+ # Demoted old-index tables still awaiting teardown: an optimize-storage run stopped after its backfill.+ stats["fts_teardown_pending"] = conn.execute(+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' "+ "LIMIT 1").fetchone() is not Nonestats["fts_rebuild_high_water"] = high_water = _meta_int("fts_rebuild_high_water")stats["fts_rebuild_progress"] = progress = _meta_int("fts_rebuild_progress")stats["fts_rebuild_pending"] = False if high_water is None else (progress or 0) < high_waterdiff --git a/hermes_state_fts.py b/hermes_state_fts.pyindex 8270560aea3..0829be597d7 100644--- a/hermes_state_fts.py+++ b/hermes_state_fts.py@@ -4,6 +4,7 @@ FTS-scoped corruption detection and the atomic fail-open trigger detach."""import loggingimport os+import reimport sqlite3import timefrom pathlib import Path@@ -130,38 +131,101 @@ def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:_FTS_DDL_TABLES = ("messages_fts", "messages_fts_trigram")-def _expected_vtable_sql(ddl: str, table: str) -> str:- """The stored-form ``CREATE VIRTUAL TABLE`` for *table* that *ddl* would create (``IF NOT EXISTS`` and- ``;`` dropped, whitespace collapsed), i.e. what ``sqlite_master.sql`` holds once the DDL has run."""- marker = f"CREATE VIRTUAL TABLE IF NOT EXISTS {table} USING"- start = ddl.index(marker)+def _stored_form(sql: str) -> str:+ """``sqlite_master.sql`` as compared here: whitespace collapsed and the object name unquoted (an ALTER TABLE+ RENAME round trip stores it as ``"name"`` without changing the layout)."""+ return re.sub(r'^CREATE (VIRTUAL TABLE|VIEW) "([^"]+)"', r"CREATE \1 \2", " ".join((sql or "").split()))+++def _expected_stored_sql(ddl: str, kind: str, name: str) -> str:+ """The stored form of the ``CREATE <kind> IF NOT EXISTS <name>`` statement in *ddl* (``IF NOT EXISTS``+ and ``;`` dropped), i.e. what ``sqlite_master.sql`` holds once the DDL has run."""+ start = ddl.index(f"CREATE {kind} IF NOT EXISTS {name} ")end = ddl.index(";", start)- return " ".join(ddl[start:end].replace(" IF NOT EXISTS", "", 1).split())+ return _stored_form(ddl[start:end].replace(" IF NOT EXISTS", "", 1))+++def _expected_vtable_sql(ddl: str, table: str) -> str:+ return _expected_stored_sql(ddl, "VIRTUAL TABLE", table)+++# Every stored definition a released version has created, oldest first; the LAST entry is the one this code+# creates. optimize-storage rebuilds only the older entries: an unrecognised definition (a newer release's, after+# a downgrade) is left alone. Literal on purpose: a change to FTS_SQL / FTS_TRIGRAM_SQL fails+# test_the_layout_registry_ends_with_the_shipping_ddl until its new form is appended here, which keeps the+# previous form recognisable as old.+FTS_VTABLE_LAYOUTS: dict[str, tuple[str, ...]] = {+ "messages_fts": (+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content, content=messages, content_rowid=id )",+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content )",+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages', "+ "content_rowid='id' )",+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages_fts_src', "+ "content_rowid='id' )",+ ),+ "messages_fts_trigram": (+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, content=messages, content_rowid=id, "+ "tokenize='trigram' )",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tokenize='trigram' )",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, tool_calls, "+ "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, "+ "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",+ ),+}+++def _stored_vtables(cursor: sqlite3.Cursor) -> dict[str, str]:+ rows = cursor.execute(+ "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name IN (?, ?)", _FTS_DDL_TABLES).fetchall()+ return {name: _stored_form(sql) for name, sql in rows}+++def _is_older_layout(table: str, stored: str) -> bool:+ """A definition an earlier release created. The two shape checks keep every store the pre-registry code+ treated as old (any base index without ``tool_name``, any trigram with ``tool_calls``) on the opt-in path."""+ if stored in FTS_VTABLE_LAYOUTS[table][:-1]:+ return True+ if table == "messages_fts":+ return "tool_name" not in stored+ return "tool_calls" in stored.lower()+++def base_index_realigned_on_open(stored: str) -> bool:+ """External-content base index not reading ``messages_fts_src``: the next read-write open rebuilds it from+ the current DDL (``_migrate_misaligned_fts_source``), so it is never opt-in work."""+ return "tool_name" in stored and "messages_fts_src" not in storeddef fts_layout_drift(cursor: sqlite3.Cursor) -> list[str]:- """FTS tables whose stored definition differs from what this code creates; empty when current.-- The one structural answer to "does optimize-storage have layout work": the update notice, doctor, the- open-time stamp and optimize-storage all read it, so a layout bump needs no new per-version predicate- and the ``fts_storage_version`` stamp is only ever written over tables that match it. An absent table- is not drift (the trigram index is optional; the base index is created on open), and neither is an- external-content base index over raw ``messages``: the next read-write open replaces it with the current- DDL (``_migrate_misaligned_fts_source``). The cjk index is left out: it exists only where its tokenizer- loads and owns its own stale/backfill markers.+ """FTS tables on an older release's definition that only ``optimize-storage`` replaces; empty otherwise.++ Shared by the update notice, doctor and optimize-storage's demote decision. An absent table is not drift+ (the trigram index is optional; the base index is created on open), nor is a base index the next open+ realigns, nor a definition this code does not recognise. The cjk index owns its own markers."""- expected = {"messages_fts": _expected_vtable_sql(FTS_SQL, "messages_fts"),- "messages_fts_trigram": _expected_vtable_sql(FTS_TRIGRAM_SQL, "messages_fts_trigram")}- drift = []- for table in _FTS_DDL_TABLES:- row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", (table,)).fetchone()- if row is None:- continue- stored = " ".join((row[0] or "").split())- realigned_on_open = table == "messages_fts" and "tool_name" in stored and "messages_fts_src" not in stored- if stored != expected[table] and not realigned_on_open:- drift.append(table)- return drift+ return [table for table, stored in _stored_vtables(cursor).items()+ if _is_older_layout(table, stored)+ and not (table == "messages_fts" and base_index_realigned_on_open(stored))]+++def fts_layout_current(cursor: sqlite3.Cursor) -> bool:+ """True when every present FTS table, and the base index's source view, is exactly what this code creates:+ the only state the ``fts_storage_version`` stamp may be written over."""+ stored = _stored_vtables(cursor)+ if any(sql != FTS_VTABLE_LAYOUTS[table][-1] for table, sql in stored.items()):+ return False+ if "messages_fts" not in stored:+ return True+ row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'view' AND name = 'messages_fts_src'").fetchone()+ return row is not None and _stored_form(row[0]) == _expected_stored_sql(FTS_SQL, "VIEW", "messages_fts_src")+++def base_source_view_is_stale(cursor: sqlite3.Cursor) -> bool:+ """``messages_fts_src`` exists but differs from what this code creates (``CREATE VIEW IF NOT EXISTS`` never+ replaces it)."""+ row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'view' AND name = 'messages_fts_src'").fetchone()+ return row is not None and _stored_form(row[0]) != _expected_stored_sql(FTS_SQL, "VIEW", "messages_fts_src")# FTS5 shadow tables the virtual-table engine owns. `sqlite3 .recover` re-emits themdiff --git a/hermes_state_schema.py b/hermes_state_schema.pyindex 289c7573178..afb64314c65 100644--- a/hermes_state_schema.py+++ b/hermes_state_schema.py@@ -26,7 +26,7 @@ from hermes_state_common import (LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,)-from hermes_state_fts import _drop_orphan_fts_shadow_tables, fts_layout_drift+from hermes_state_fts import _drop_orphan_fts_shadow_tables, base_source_view_is_stale, fts_layout_currentfrom hermes_state_holders import _read_proc_argvfrom hermes_state_search import _delete_meta, _meta_rowfrom hermes_state_errors import is_sqlite_lock_error@@ -358,7 +358,8 @@ class SessionSchemaMixin:optimize path)."""if legacy or not self._sqlite_table_exists(cursor, "messages_fts"):return- if not self._fts_index_is_misaligned_source(cursor):+ stale_view = base_source_view_is_stale(cursor)+ if not stale_view and not self._fts_index_is_misaligned_source(cursor):returnhas_messages = cursor.execute("SELECT 1 FROM messages LIMIT 1").fetchone() is not None@@ -366,9 +367,18 @@ class SessionSchemaMixin:for name in _FTS_BASE_TRIGGERS:cursor.execute(f"DROP TRIGGER IF EXISTS {name}")cursor.execute("DROP TABLE IF EXISTS messages_fts")+ if stale_view:+ # CREATE VIEW IF NOT EXISTS never replaces a differing view; the index is rebuilt from it below.+ cursor.execute("DROP VIEW messages_fts_src")self._ensure_fts_schema(cursor, "messages_fts", FTS_SQL)if has_messages:cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")+ # The cleared markers below are shared with the trigram backfill: finish that index too, or rows+ # an interrupted backfill had not reached stay out of it with nothing left to resume them.+ if self._sqlite_table_exists(cursor, "messages_fts_trigram") and _meta_row(+ cursor, "fts_rebuild_high_water"+ ) is not None:+ cursor.execute("INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')")cursor.execute(_CLEAR_REBUILD_MARKERS_SQL)cursor.execute(_DROP_RETIRED_TOOL_HIGH_WATER_SQL)@@ -1007,17 +1017,18 @@ class SessionSchemaMixin:self._conn.commit()def _stamp_settled_fts_layout(self, cursor: sqlite3.Cursor) -> None:- """Write ``fts_storage_version`` once the stored layout is current and nothing is in flight.+ """Write ``fts_storage_version`` once the stored layout is exactly current and nothing is in flight.The only open-time writer of the stamp, run after _init_fts so a fresh store is stamped on creation- and a realigned base index is stamped only when the trigram also matches (``fts_layout_drift``). An- interrupted optimize (markers, trash, an empty external index against non-empty messages) keeps the- stamp unwritten so the resume offer stays alive. Reads before writing: a settled store never takes- the write lock here.+ and a realigned base index is stamped only when the trigram also matches. A definition this code does+ not recognise (a newer release's) is never stamped down to this release's version. An interrupted+ optimize (markers, trash, an empty external index against non-empty messages) keeps the stamp+ unwritten so the resume offer stays alive. Reads before writing: a settled store never takes the write+ lock here."""if (self._fts_stale- or fts_layout_drift(cursor)+ or not fts_layout_current(cursor)or _meta_row(cursor, "fts_rebuild_high_water") is not Noneor self._has_fts_trash(cursor)or self._fts_external_index_empty_with_messages(cursor)diff --git a/hermes_state_search.py b/hermes_state_search.pyindex c4b8ea84cfc..5b2a1ddd801 100644--- a/hermes_state_search.py+++ b/hermes_state_search.py@@ -19,7 +19,7 @@ from hermes_state_common import (MAX_FTS5_QUERY_CHARS, SCHEMA_VERSION, _FTS_CJK_TRIGGERS,escape_like as _escape_like, fts_rebuild_admission, fts_trigram_session_sql, routed_sessions_setting,)-from hermes_state_fts import fts_layout_drift+from hermes_state_fts import fts_layout_current# Pre-split logger identity so log filtering/capture is unchanged.logger = logging.getLogger("hermes_state")@@ -644,7 +644,7 @@ class SessionSearchMixin:return "teardown_incomplete"if self._fts_external_index_empty_with_messages(conn):return "backfill_incomplete"- if fts_layout_drift(conn):+ if not fts_layout_current(conn):return "layout_drift"self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn)_delete_meta(conn, "fts_optimize_available")