Skip to content

The agent that grows with you

fix(state): recognise old FTS layouts by name; stamp only the exact current one

yoniebans281d5cead05d
diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py
index b9ec8522d41..fca53626866 100644
--- a/hermes_cli/config_defaults.py
+++ b/hermes_cli/config_defaults.py
@@ -2310,8 +2310,8 @@ DEFAULT_CONFIG = {
# Notice about the compact FTS layout (reclaims ~60%+ of state.db). OPT-IN: legacy indexes
# stay until `hermes sessions optimize-storage` runs, since the rebuild is disk-heavy on
# large DBs. advise = `hermes update` prints a one-line notice with reclaimable size when a
- # legacy index is detected; require = shown as a REQUIRED upgrade (tooling may gate on it);
- # off = none.
+ # legacy index is detected (a later layout change gets a rebuild notice without a size estimate);
+ # require = the same notices worded as REQUIRED (tooling may gate on it); off = none.
"fts_optimize_notice": "advise",
# CJK-bigram search index (messages_fts_cjk). When the extension is built
# (native/fts5_cjk/build.sh → ~/.hermes/lib/libfts5_cjk.so), 1-2 char CJK terms get exact
diff --git a/hermes_cli/doctor_state.py b/hermes_cli/doctor_state.py
index c796add0c79..b74ce1399b9 100644
--- a/hermes_cli/doctor_state.py
+++ b/hermes_cli/doctor_state.py
@@ -93,11 +93,12 @@ def _render_state_db_stats(stats: dict, holders=None, host_note: str = "") -> li
"(stop the listed processes; the host gateway's own retry then rebuilds, or run "
"'hermes sessions optimize-storage' with every holder stopped)"))
# Oversized DB: suggest auto_prune, plus the offline optimize-storage pass when the FTS rebuild is
- # pending OR the stored FTS layout structurally needs upgrading (not just a lagging marker).
+ # pending, an earlier run stopped before tearing down the old index, or the stored FTS layout is an
+ # older release's (not just a lagging marker).
if logical is not None and logical > STATE_DB_SIZE_WARN_BYTES:
detail = "consider enabling sessions.auto_prune in config.yaml to bound growth"
structural_fts_upgrade = bool(stats.get("fts_storage_upgrade_needed"))
- if stats.get("fts_rebuild_pending") or structural_fts_upgrade:
+ if stats.get("fts_rebuild_pending") or stats.get("fts_teardown_pending") or structural_fts_upgrade:
detail += "; run 'hermes sessions optimize-storage' offline (with the host gateway stopped) to compact FTS storage"
lines.append(("warn", f"state.db is large ({_human_bytes(logical)})", f"({detail})"))
# WAL runaway is deliberately NOT warned here: _state_db_wal already warns above 50 MB and offers --fix.
diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py
index e6221d01368..079a99591ba 100644
--- a/hermes_cli/update_cmd_maint.py
+++ b/hermes_cli/update_cmd_maint.py
@@ -198,11 +198,14 @@ def _print_fts_optimize_available_notice() -> None:
f"of your current {size_gb:.1f} GB."
)
else:
- print("◆ Search index update available")
+ if mode == "require":
+ print("◆ Search index rebuild required by your notice policy")
+ else:
+ print("◆ Search index update available")
print(
- f" Your search index is on the compact layout from an earlier "
- f"optimization, but a later release changed it. This is a smaller "
- f"follow-up rebuild than the first optimization."
+ " Your search index is on the compact layout from an earlier "
+ "optimization, but a later release changed it. Running the "
+ "command again rebuilds it in the current layout."
)
print(" Run when convenient: hermes sessions optimize-storage")
print(
diff --git a/hermes_state_dbfile.py b/hermes_state_dbfile.py
index a5ae94188f5..a6dff2d948e 100644
--- a/hermes_state_dbfile.py
+++ b/hermes_state_dbfile.py
@@ -752,7 +752,8 @@ def collect_state_db_stats(db_path: Path) -> dict[str, Any]:
from hermes_state import _connect_tracked_db
stats: dict[str, Any] = dict.fromkeys((
"page_count", "page_size", "freelist_count", "logical_size_bytes", "wal_size_bytes", "journal_mode",
- "messages", "sessions", "fts_tables", "fts_storage_version", "fts_storage_upgrade_needed", "fts_rebuild_pending",
+ "messages", "sessions", "fts_tables", "fts_storage_version", "fts_storage_upgrade_needed",
+ "fts_teardown_pending", "fts_rebuild_pending",
"fts_rebuild_high_water", "fts_rebuild_progress", "fts_rebuild_deferral"))
# WAL sidecar size needs no connection at all.
with contextlib.suppress(OSError):
@@ -803,6 +804,11 @@ def collect_state_db_stats(db_path: Path) -> dict[str, Any]:
with contextlib.suppress(Exception):
from hermes_state_fts import fts_layout_drift
stats["fts_storage_upgrade_needed"] = bool(fts_layout_drift(conn.cursor()))
+ with contextlib.suppress(Exception):
+ # Demoted old-index tables still awaiting teardown: an optimize-storage run stopped after its backfill.
+ stats["fts_teardown_pending"] = conn.execute(
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' "
+ "LIMIT 1").fetchone() is not None
stats["fts_rebuild_high_water"] = high_water = _meta_int("fts_rebuild_high_water")
stats["fts_rebuild_progress"] = progress = _meta_int("fts_rebuild_progress")
stats["fts_rebuild_pending"] = False if high_water is None else (progress or 0) < high_water
diff --git a/hermes_state_fts.py b/hermes_state_fts.py
index 8270560aea3..0829be597d7 100644
--- a/hermes_state_fts.py
+++ b/hermes_state_fts.py
@@ -4,6 +4,7 @@ FTS-scoped corruption detection and the atomic fail-open trigger detach."""
import logging
import os
+import re
import sqlite3
import time
from pathlib import Path
@@ -130,38 +131,101 @@ def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
_FTS_DDL_TABLES = ("messages_fts", "messages_fts_trigram")
-def _expected_vtable_sql(ddl: str, table: str) -> str:
- """The stored-form ``CREATE VIRTUAL TABLE`` for *table* that *ddl* would create (``IF NOT EXISTS`` and
- ``;`` dropped, whitespace collapsed), i.e. what ``sqlite_master.sql`` holds once the DDL has run."""
- marker = f"CREATE VIRTUAL TABLE IF NOT EXISTS {table} USING"
- start = ddl.index(marker)
+def _stored_form(sql: str) -> str:
+ """``sqlite_master.sql`` as compared here: whitespace collapsed and the object name unquoted (an ALTER TABLE
+ RENAME round trip stores it as ``"name"`` without changing the layout)."""
+ return re.sub(r'^CREATE (VIRTUAL TABLE|VIEW) "([^"]+)"', r"CREATE \1 \2", " ".join((sql or "").split()))
+
+
+def _expected_stored_sql(ddl: str, kind: str, name: str) -> str:
+ """The stored form of the ``CREATE <kind> IF NOT EXISTS <name>`` statement in *ddl* (``IF NOT EXISTS``
+ and ``;`` dropped), i.e. what ``sqlite_master.sql`` holds once the DDL has run."""
+ start = ddl.index(f"CREATE {kind} IF NOT EXISTS {name} ")
end = ddl.index(";", start)
- return " ".join(ddl[start:end].replace(" IF NOT EXISTS", "", 1).split())
+ return _stored_form(ddl[start:end].replace(" IF NOT EXISTS", "", 1))
+
+
+def _expected_vtable_sql(ddl: str, table: str) -> str:
+ return _expected_stored_sql(ddl, "VIRTUAL TABLE", table)
+
+
+# Every stored definition a released version has created, oldest first; the LAST entry is the one this code
+# creates. optimize-storage rebuilds only the older entries: an unrecognised definition (a newer release's, after
+# a downgrade) is left alone. Literal on purpose: a change to FTS_SQL / FTS_TRIGRAM_SQL fails
+# test_the_layout_registry_ends_with_the_shipping_ddl until its new form is appended here, which keeps the
+# previous form recognisable as old.
+FTS_VTABLE_LAYOUTS: dict[str, tuple[str, ...]] = {
+ "messages_fts": (
+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content, content=messages, content_rowid=id )",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content )",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages', "
+ "content_rowid='id' )",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages_fts_src', "
+ "content_rowid='id' )",
+ ),
+ "messages_fts_trigram": (
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, content=messages, content_rowid=id, "
+ "tokenize='trigram' )",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tokenize='trigram' )",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, tool_calls, "
+ "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, "
+ "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",
+ ),
+}
+
+
+def _stored_vtables(cursor: sqlite3.Cursor) -> dict[str, str]:
+ rows = cursor.execute(
+ "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name IN (?, ?)", _FTS_DDL_TABLES).fetchall()
+ return {name: _stored_form(sql) for name, sql in rows}
+
+
+def _is_older_layout(table: str, stored: str) -> bool:
+ """A definition an earlier release created. The two shape checks keep every store the pre-registry code
+ treated as old (any base index without ``tool_name``, any trigram with ``tool_calls``) on the opt-in path."""
+ if stored in FTS_VTABLE_LAYOUTS[table][:-1]:
+ return True
+ if table == "messages_fts":
+ return "tool_name" not in stored
+ return "tool_calls" in stored.lower()
+
+
+def base_index_realigned_on_open(stored: str) -> bool:
+ """External-content base index not reading ``messages_fts_src``: the next read-write open rebuilds it from
+ the current DDL (``_migrate_misaligned_fts_source``), so it is never opt-in work."""
+ return "tool_name" in stored and "messages_fts_src" not in stored
def fts_layout_drift(cursor: sqlite3.Cursor) -> list[str]:
- """FTS tables whose stored definition differs from what this code creates; empty when current.
-
- The one structural answer to "does optimize-storage have layout work": the update notice, doctor, the
- open-time stamp and optimize-storage all read it, so a layout bump needs no new per-version predicate
- and the ``fts_storage_version`` stamp is only ever written over tables that match it. An absent table
- is not drift (the trigram index is optional; the base index is created on open), and neither is an
- external-content base index over raw ``messages``: the next read-write open replaces it with the current
- DDL (``_migrate_misaligned_fts_source``). The cjk index is left out: it exists only where its tokenizer
- loads and owns its own stale/backfill markers.
+ """FTS tables on an older release's definition that only ``optimize-storage`` replaces; empty otherwise.
+
+ Shared by the update notice, doctor and optimize-storage's demote decision. An absent table is not drift
+ (the trigram index is optional; the base index is created on open), nor is a base index the next open
+ realigns, nor a definition this code does not recognise. The cjk index owns its own markers.
"""
- expected = {"messages_fts": _expected_vtable_sql(FTS_SQL, "messages_fts"),
- "messages_fts_trigram": _expected_vtable_sql(FTS_TRIGRAM_SQL, "messages_fts_trigram")}
- drift = []
- for table in _FTS_DDL_TABLES:
- row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", (table,)).fetchone()
- if row is None:
- continue
- stored = " ".join((row[0] or "").split())
- realigned_on_open = table == "messages_fts" and "tool_name" in stored and "messages_fts_src" not in stored
- if stored != expected[table] and not realigned_on_open:
- drift.append(table)
- return drift
+ return [table for table, stored in _stored_vtables(cursor).items()
+ if _is_older_layout(table, stored)
+ and not (table == "messages_fts" and base_index_realigned_on_open(stored))]
+
+
+def fts_layout_current(cursor: sqlite3.Cursor) -> bool:
+ """True when every present FTS table, and the base index's source view, is exactly what this code creates:
+ the only state the ``fts_storage_version`` stamp may be written over."""
+ stored = _stored_vtables(cursor)
+ if any(sql != FTS_VTABLE_LAYOUTS[table][-1] for table, sql in stored.items()):
+ return False
+ if "messages_fts" not in stored:
+ return True
+ row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'view' AND name = 'messages_fts_src'").fetchone()
+ return row is not None and _stored_form(row[0]) == _expected_stored_sql(FTS_SQL, "VIEW", "messages_fts_src")
+
+
+def base_source_view_is_stale(cursor: sqlite3.Cursor) -> bool:
+ """``messages_fts_src`` exists but differs from what this code creates (``CREATE VIEW IF NOT EXISTS`` never
+ replaces it)."""
+ row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'view' AND name = 'messages_fts_src'").fetchone()
+ return row is not None and _stored_form(row[0]) != _expected_stored_sql(FTS_SQL, "VIEW", "messages_fts_src")
# FTS5 shadow tables the virtual-table engine owns. `sqlite3 .recover` re-emits them
diff --git a/hermes_state_schema.py b/hermes_state_schema.py
index 289c7573178..afb64314c65 100644
--- a/hermes_state_schema.py
+++ b/hermes_state_schema.py
@@ -26,7 +26,7 @@ from hermes_state_common import (
LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,
SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,
)
-from hermes_state_fts import _drop_orphan_fts_shadow_tables, fts_layout_drift
+from hermes_state_fts import _drop_orphan_fts_shadow_tables, base_source_view_is_stale, fts_layout_current
from hermes_state_holders import _read_proc_argv
from hermes_state_search import _delete_meta, _meta_row
from hermes_state_errors import is_sqlite_lock_error
@@ -358,7 +358,8 @@ class SessionSchemaMixin:
optimize path)."""
if legacy or not self._sqlite_table_exists(cursor, "messages_fts"):
return
- if not self._fts_index_is_misaligned_source(cursor):
+ stale_view = base_source_view_is_stale(cursor)
+ if not stale_view and not self._fts_index_is_misaligned_source(cursor):
return
has_messages = cursor.execute("SELECT 1 FROM messages LIMIT 1").fetchone() is not None
@@ -366,9 +367,18 @@ class SessionSchemaMixin:
for name in _FTS_BASE_TRIGGERS:
cursor.execute(f"DROP TRIGGER IF EXISTS {name}")
cursor.execute("DROP TABLE IF EXISTS messages_fts")
+ if stale_view:
+ # CREATE VIEW IF NOT EXISTS never replaces a differing view; the index is rebuilt from it below.
+ cursor.execute("DROP VIEW messages_fts_src")
self._ensure_fts_schema(cursor, "messages_fts", FTS_SQL)
if has_messages:
cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
+ # The cleared markers below are shared with the trigram backfill: finish that index too, or rows
+ # an interrupted backfill had not reached stay out of it with nothing left to resume them.
+ if self._sqlite_table_exists(cursor, "messages_fts_trigram") and _meta_row(
+ cursor, "fts_rebuild_high_water"
+ ) is not None:
+ cursor.execute("INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')")
cursor.execute(_CLEAR_REBUILD_MARKERS_SQL)
cursor.execute(_DROP_RETIRED_TOOL_HIGH_WATER_SQL)
@@ -1007,17 +1017,18 @@ class SessionSchemaMixin:
self._conn.commit()
def _stamp_settled_fts_layout(self, cursor: sqlite3.Cursor) -> None:
- """Write ``fts_storage_version`` once the stored layout is current and nothing is in flight.
+ """Write ``fts_storage_version`` once the stored layout is exactly current and nothing is in flight.
The only open-time writer of the stamp, run after _init_fts so a fresh store is stamped on creation
- and a realigned base index is stamped only when the trigram also matches (``fts_layout_drift``). An
- interrupted optimize (markers, trash, an empty external index against non-empty messages) keeps the
- stamp unwritten so the resume offer stays alive. Reads before writing: a settled store never takes
- the write lock here.
+ and a realigned base index is stamped only when the trigram also matches. A definition this code does
+ not recognise (a newer release's) is never stamped down to this release's version. An interrupted
+ optimize (markers, trash, an empty external index against non-empty messages) keeps the stamp
+ unwritten so the resume offer stays alive. Reads before writing: a settled store never takes the write
+ lock here.
"""
if (
self._fts_stale
- or fts_layout_drift(cursor)
+ or not fts_layout_current(cursor)
or _meta_row(cursor, "fts_rebuild_high_water") is not None
or self._has_fts_trash(cursor)
or self._fts_external_index_empty_with_messages(cursor)
diff --git a/hermes_state_search.py b/hermes_state_search.py
index c4b8ea84cfc..5b2a1ddd801 100644
--- a/hermes_state_search.py
+++ b/hermes_state_search.py
@@ -19,7 +19,7 @@ from hermes_state_common import (
MAX_FTS5_QUERY_CHARS, SCHEMA_VERSION, _FTS_CJK_TRIGGERS,
escape_like as _escape_like, fts_rebuild_admission, fts_trigram_session_sql, routed_sessions_setting,
)
-from hermes_state_fts import fts_layout_drift
+from hermes_state_fts import fts_layout_current
# Pre-split logger identity so log filtering/capture is unchanged.
logger = logging.getLogger("hermes_state")
@@ -644,7 +644,7 @@ class SessionSearchMixin:
return "teardown_incomplete"
if self._fts_external_index_empty_with_messages(conn):
return "backfill_incomplete"
- if fts_layout_drift(conn):
+ if not fts_layout_current(conn):
return "layout_drift"
self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn)
_delete_meta(conn, "fts_optimize_available")