Skip to content

The agent that grows with you

fix(state): one structural FTS layout-drift answer for the stamp, doctor and the update notice

yoniebansf42c307ef871
diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py
index 1e2ab5b2c24..e6221d01368 100644
--- a/hermes_cli/update_cmd_maint.py
+++ b/hermes_cli/update_cmd_maint.py
@@ -134,15 +134,14 @@ def _print_fts_optimize_available_notice() -> None:
if size_gb < 0.5:
return
db = None
- needs_upgrade = False
+ legacy_inline = False
+ drift: list[str] = []
try:
db = SessionDB(db_path=db_path, read_only=True)
# read_only opens skip schema init; probe the stored layout directly.
- row = db._conn.execute(
- "SELECT sql FROM sqlite_master "
- "WHERE type = 'table' AND name = 'messages_fts'"
- ).fetchone()
- needs_upgrade = bool(row) and db._db_needs_fts_storage_upgrade(db._conn)
+ from hermes_state_fts import fts_layout_drift
+ drift = fts_layout_drift(db._conn)
+ legacy_inline = db._db_has_legacy_inline_fts(db._conn)
# Interrupted optimize-storage: v23 table shape but backfill markers / trash
# tables remain. Re-running resumes it, so offer the command again.
interrupted = bool(
@@ -165,7 +164,7 @@ def _print_fts_optimize_available_notice() -> None:
if db is not None:
with suppress(Exception):
db.close()
- if not needs_upgrade and not interrupted:
+ if not drift and not interrupted:
return # current layout already present (fresh/optimized)
if interrupted:
@@ -179,22 +178,31 @@ def _print_fts_optimize_available_notice() -> None:
print(" hermes sessions optimize-storage")
return
- est_reclaim = size_gb * 0.6
print()
- if mode == "require":
- print("◆ Session database upgrade required")
- print(
- f" Your search index uses the OLD storage layout and should be "
- f"upgraded. The new layout typically frees ~60% of state.db "
- f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is "
- f"required for continued optimal operation."
- )
+ if legacy_inline:
+ # The ~60% figure was measured for this transition only (inline copies of every message dropped).
+ est_reclaim = size_gb * 0.6
+ if mode == "require":
+ print("◆ Session database upgrade required")
+ print(
+ f" Your search index uses the OLD storage layout and should be "
+ f"upgraded. The new layout typically frees ~60% of state.db "
+ f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is "
+ f"required for continued optimal operation."
+ )
+ else:
+ print("◆ Reclaim ~60% of your session database disk")
+ print(
+ f" Your search index uses the old storage layout. Upgrading it "
+ f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB "
+ f"of your current {size_gb:.1f} GB."
+ )
else:
- print("◆ Reclaim ~60% of your session database disk")
+ print("◆ Search index update available")
print(
- f" Your search index uses the old storage layout. Upgrading it "
- f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB "
- f"of your current {size_gb:.1f} GB."
+ f" Your search index is on the compact layout from an earlier "
+ f"optimization, but a later release changed it. This is a smaller "
+ f"follow-up rebuild than the first optimization."
)
print(" Run when convenient: hermes sessions optimize-storage")
print(
diff --git a/hermes_state_dbfile.py b/hermes_state_dbfile.py
index 483683ffb85..a5ae94188f5 100644
--- a/hermes_state_dbfile.py
+++ b/hermes_state_dbfile.py
@@ -801,8 +801,8 @@ def collect_state_db_stats(db_path: Path) -> dict[str, Any]:
# Raw state_meta reads — cheap, and independent of SessionDB.
stats["fts_storage_version"] = _meta_int("fts_storage_version")
with contextlib.suppress(Exception):
- from hermes_state_fts import SessionFtsSetupMixin
- stats["fts_storage_upgrade_needed"] = SessionFtsSetupMixin._db_needs_fts_storage_upgrade(conn.cursor())
+ from hermes_state_fts import fts_layout_drift
+ stats["fts_storage_upgrade_needed"] = bool(fts_layout_drift(conn.cursor()))
stats["fts_rebuild_high_water"] = high_water = _meta_int("fts_rebuild_high_water")
stats["fts_rebuild_progress"] = progress = _meta_int("fts_rebuild_progress")
stats["fts_rebuild_pending"] = False if high_water is None else (progress or 0) < high_water
diff --git a/hermes_state_fts.py b/hermes_state_fts.py
index 3ced10cb2c2..f4d0531ae17 100644
--- a/hermes_state_fts.py
+++ b/hermes_state_fts.py
@@ -10,8 +10,8 @@ from pathlib import Path
from typing import Sequence
from hermes_constants import get_hermes_home
-from hermes_state_common import (FTS_CJK_STALE_KEY, FTS_STALE_KEY, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS,
- routed_sessions_setting)
+from hermes_state_common import (FTS_CJK_STALE_KEY, FTS_SQL, FTS_STALE_KEY, FTS_TRIGRAM_SQL, _FTS_CJK_TRIGGERS,
+ _FTS_TRIGGERS, routed_sessions_setting)
from hermes_state_errors import is_fts_scoped_corruption_error, is_sqlite_lock_error
# caplog tests pin the "hermes_state" logger name.
@@ -127,6 +127,37 @@ def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
return False
+_FTS_DDL_TABLES = ("messages_fts", "messages_fts_trigram")
+
+
+def _expected_vtable_sql(ddl: str, table: str) -> str:
+ """The stored-form ``CREATE VIRTUAL TABLE`` for *table* that *ddl* would create (``IF NOT EXISTS`` and
+ ``;`` dropped, whitespace collapsed), i.e. what ``sqlite_master.sql`` holds once the DDL has run."""
+ marker = f"CREATE VIRTUAL TABLE IF NOT EXISTS {table} USING"
+ start = ddl.index(marker)
+ end = ddl.index(";", start)
+ return " ".join(ddl[start:end].replace(" IF NOT EXISTS", "", 1).split())
+
+
+def fts_layout_drift(cursor: sqlite3.Cursor) -> list[str]:
+ """FTS tables whose stored definition differs from what this code creates; empty when current.
+
+ The one structural answer to "does optimize-storage have layout work": the update notice, doctor, the
+ open-time stamp and optimize-storage all read it, so a layout bump needs no new per-version predicate
+ and the ``fts_storage_version`` stamp is only ever written over tables that match it. An absent table
+ is not drift (the trigram index is optional; the base index is created on open). The cjk index is
+ left out: it exists only where its tokenizer loads and owns its own stale/backfill markers.
+ """
+ expected = {"messages_fts": _expected_vtable_sql(FTS_SQL, "messages_fts"),
+ "messages_fts_trigram": _expected_vtable_sql(FTS_TRIGRAM_SQL, "messages_fts_trigram")}
+ drift = []
+ for table in _FTS_DDL_TABLES:
+ row = cursor.execute("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", (table,)).fetchone()
+ if row is not None and " ".join((row[0] or "").split()) != expected[table]:
+ drift.append(table)
+ return drift
+
+
# FTS5 shadow tables the virtual-table engine owns. `sqlite3 .recover` re-emits them
# as ordinary tables but cannot re-emit the CREATE VIRTUAL TABLE row, so every later
# CREATE VIRTUAL TABLE fails with "fts5: error creating shadow table <name>: table
@@ -210,8 +241,8 @@ class SessionFtsSetupMixin:
@classmethod
def _db_needs_fts_storage_upgrade(cls, cursor: sqlite3.Cursor) -> bool:
- """True when the current FTS storage layout should be treated as stale (optimize-storage has work)."""
- return cls._db_has_legacy_inline_fts(cursor) or cls._db_has_trigram_tool_calls_projection(cursor)
+ """True when optimize-storage has layout work: a stored FTS vtable differs from what this code creates."""
+ return bool(fts_layout_drift(cursor))
def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None:
"""Log once that the trigram tokenizer is missing; base FTS5 stays enabled."""
diff --git a/hermes_state_schema.py b/hermes_state_schema.py
index edde611e980..289c7573178 100644
--- a/hermes_state_schema.py
+++ b/hermes_state_schema.py
@@ -26,7 +26,7 @@ from hermes_state_common import (
LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,
SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,
)
-from hermes_state_fts import _drop_orphan_fts_shadow_tables
+from hermes_state_fts import _drop_orphan_fts_shadow_tables, fts_layout_drift
from hermes_state_holders import _read_proc_argv
from hermes_state_search import _delete_meta, _meta_row
from hermes_state_errors import is_sqlite_lock_error
@@ -371,7 +371,6 @@ class SessionSchemaMixin:
cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
cursor.execute(_CLEAR_REBUILD_MARKERS_SQL)
cursor.execute(_DROP_RETIRED_TOOL_HIGH_WATER_SQL)
- cursor.execute(_STATE_META_UPSERT_SQL, ("fts_storage_version", str(FTS_STORAGE_VERSION)))
if not has_messages:
# Nothing indexed and nothing to index: swap the shape in place, no rebuild authority needed.
@@ -1004,8 +1003,32 @@ class SessionSchemaMixin:
self._ensure_unique_title_index(cursor)
if fts5_available:
self._init_fts(cursor)
+ self._stamp_settled_fts_layout(cursor)
self._conn.commit()
+ def _stamp_settled_fts_layout(self, cursor: sqlite3.Cursor) -> None:
+ """Write ``fts_storage_version`` once the stored layout is current and nothing is in flight.
+
+ The only open-time writer of the stamp, run after _init_fts so a fresh store is stamped on creation
+ and a realigned base index is stamped only when the trigram also matches (``fts_layout_drift``). An
+ interrupted optimize (markers, trash, an empty external index against non-empty messages) keeps the
+ stamp unwritten so the resume offer stays alive. Reads before writing: a settled store never takes
+ the write lock here.
+ """
+ if (
+ self._fts_stale
+ or fts_layout_drift(cursor)
+ or _meta_row(cursor, "fts_rebuild_high_water") is not None
+ or self._has_fts_trash(cursor)
+ or self._fts_external_index_empty_with_messages(cursor)
+ ):
+ return
+ if cursor.execute(
+ "SELECT 1 FROM state_meta WHERE key = 'fts_storage_version' AND value = ? LIMIT 1",
+ (str(FTS_STORAGE_VERSION),),
+ ).fetchone() is None:
+ cursor.execute(_STATE_META_UPSERT_SQL, ("fts_storage_version", str(FTS_STORAGE_VERSION)))
+
def _run_data_migrations(self, cursor: sqlite3.Cursor, current_version: int, fts5_available: bool) -> None:
"""Version-gated chain for DATA migrations only (row backfills); column additions never
belong here. Advances schema_version at the end unless FTS5 is unavailable."""
@@ -1069,10 +1092,7 @@ class SessionSchemaMixin:
# Rebuild once so rows indexed by older view/trigger definitions do not linger.
fts_migrations_complete = self._migrate_trigram_cron_exclusion(cursor)
- # Stamp the FTS layout version (fresh/optimized DBs); a legacy DB keeps its absent/0
- # marker until optimize-storage runs. An INTERRUPTED optimize (markers, trash, or an
- # empty external index against non-empty messages) is NOT stamped: the marker is the
- # source of truth for "fully optimized" and keeps the resume offer alive.
+ # The FTS layout stamp is written by _stamp_settled_fts_layout after _init_fts, not here.
# v23: FTS storage redesign (issues #22478, #43690, #55233). The v11 inline-mode FTS tables each
# store a full private copy of every message (content || tool_name || tool_calls), and the trigram
# index additionally covers role='tool' rows (~90% of message bytes: base64 payloads, file dumps) at
@@ -1089,23 +1109,6 @@ class SessionSchemaMixin:
# marker (see _fts_storage_version / SETTLE below), so schema_version advances to SCHEMA_VERSION
# here like every other migration — future v24+ migrations land automatically for legacy-FTS users
# too. Only the FTS *layout* waits for opt-in.
- if (
- fts5_available
- and not self._db_needs_fts_storage_upgrade(cursor)
- and cursor.execute(
- "SELECT 1 FROM state_meta WHERE key = 'fts_rebuild_high_water' LIMIT 1"
- ).fetchone() is None
- and not self._has_fts_trash(cursor)
- and not self._fts_external_index_empty_with_messages(cursor)
- ):
- # Stamp only when it would change something: on a settled DB every condition above
- # already holds, and re-writing the same value takes the write lock on every open.
- if cursor.execute(
- "SELECT 1 FROM state_meta WHERE key = 'fts_storage_version' AND value = ? LIMIT 1",
- (str(FTS_STORAGE_VERSION),),
- ).fetchone() is None:
- self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor)
-
# v32: nothing to run here. The declared tables/columns #106742 added under v31 were already
# created by _init_schema's executescript(SCHEMA_SQL) + _reconcile_columns() before this chain;
# the version only records that the store has them (#140315).
diff --git a/hermes_state_search.py b/hermes_state_search.py
index 95caec48254..c4b8ea84cfc 100644
--- a/hermes_state_search.py
+++ b/hermes_state_search.py
@@ -19,6 +19,7 @@ from hermes_state_common import (
MAX_FTS5_QUERY_CHARS, SCHEMA_VERSION, _FTS_CJK_TRIGGERS,
escape_like as _escape_like, fts_rebuild_admission, fts_trigram_session_sql, routed_sessions_setting,
)
+from hermes_state_fts import fts_layout_drift
# Pre-split logger identity so log filtering/capture is unchanged.
logger = logging.getLogger("hermes_state")
@@ -643,6 +644,8 @@ class SessionSearchMixin:
return "teardown_incomplete"
if self._fts_external_index_empty_with_messages(conn):
return "backfill_incomplete"
+ if fts_layout_drift(conn):
+ return "layout_drift"
self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn)
_delete_meta(conn, "fts_optimize_available")
conn.execute("UPDATE schema_version SET version = ? WHERE version < ?", (SCHEMA_VERSION, SCHEMA_VERSION))
diff --git a/tests/hermes_cli/test_fts_optimize_notice.py b/tests/hermes_cli/test_fts_optimize_notice.py
index a8b8ea36c27..01ce33348ff 100644
--- a/tests/hermes_cli/test_fts_optimize_notice.py
+++ b/tests/hermes_cli/test_fts_optimize_notice.py
@@ -33,6 +33,7 @@ def test_update_notice_offers_v1_trigram_tool_calls_rebuild(tmp_path, monkeypatc
_db_needs_fts_storage_upgrade = staticmethod(
hermes_state.SessionDB._db_needs_fts_storage_upgrade
)
+ _db_has_legacy_inline_fts = staticmethod(hermes_state.SessionDB._db_has_legacy_inline_fts)
monkeypatch.setattr(hermes_constants, "get_hermes_home", lambda: tmp_path)
monkeypatch.setattr(hermes_state, "SessionDB", FakeSessionDB)
@@ -64,6 +65,8 @@ def test_doctor_collects_structural_fts_upgrade_status(
"""Real read-only collection must ignore forged or lagging version markers."""
from hermes_cli import doctor_state
from hermes_state_dbfile import collect_state_db_stats
+ from hermes_state_fts import _expected_vtable_sql
+ from hermes_state_common import FTS_SQL, FTS_TRIGRAM_SQL
db_path = tmp_path / "state.db"
with sqlite3.connect(db_path) as conn:
@@ -75,13 +78,11 @@ def test_doctor_collects_structural_fts_upgrade_status(
if base_layout == "legacy":
conn.execute("CREATE VIRTUAL TABLE messages_fts USING fts5(content)")
elif base_layout == "current":
- conn.execute(
- "CREATE VIRTUAL TABLE messages_fts USING "
- "fts5(content, tool_name, tool_calls, content='messages')"
- )
- if trigram_layout != "absent":
- columns = "content, tool_name" + (", tool_calls" if trigram_layout == "legacy" else "")
- conn.execute(f"CREATE VIRTUAL TABLE messages_fts_trigram USING fts5({columns}, tokenize='trigram')")
+ conn.execute(_expected_vtable_sql(FTS_SQL, "messages_fts"))
+ if trigram_layout == "legacy":
+ conn.execute("CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content, tool_name, tool_calls, tokenize='trigram')")
+ elif trigram_layout == "current":
+ conn.execute(_expected_vtable_sql(FTS_TRIGRAM_SQL, "messages_fts_trigram"))
if marker is not None:
conn.execute("INSERT INTO state_meta VALUES ('fts_storage_version', ?)", (str(marker),))
@@ -104,13 +105,15 @@ def test_doctor_collects_structural_fts_upgrade_status(
def test_doctor_collects_pending_rebuild_with_settled_layout(tmp_path, monkeypatch, progress, pending):
from hermes_cli import doctor_state
from hermes_state_dbfile import collect_state_db_stats
+ from hermes_state_fts import _expected_vtable_sql
+ from hermes_state_common import FTS_SQL, FTS_TRIGRAM_SQL
db_path = tmp_path / "state.db"
with sqlite3.connect(db_path) as conn:
conn.executescript(
"CREATE TABLE state_meta (key TEXT PRIMARY KEY, value TEXT);"
- "CREATE VIRTUAL TABLE messages_fts USING fts5(content, tool_name, tool_calls);"
- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content, tool_name, tokenize='trigram');"
+ f"{_expected_vtable_sql(FTS_SQL, 'messages_fts')};"
+ f"{_expected_vtable_sql(FTS_TRIGRAM_SQL, 'messages_fts_trigram')};"
"INSERT INTO state_meta VALUES ('fts_rebuild_high_water', '10');"
)
conn.execute("INSERT INTO state_meta VALUES ('fts_rebuild_progress', ?)", (str(progress),))
diff --git a/tests/hermes_state/test_hermes_state.py b/tests/hermes_state/test_hermes_state.py
index 4946afb4f6f..61542351520 100644
--- a/tests/hermes_state/test_hermes_state.py
+++ b/tests/hermes_state/test_hermes_state.py
@@ -4705,6 +4705,9 @@ class TestFTSExternalContentMigration:
try:
db.create_session(session_id="s1", source="cli")
db.append_message("s1", role="user", content="settle guard needle")
+ # A fresh store is stamped on creation; clear it so the assertion reads settle's own write.
+ db._conn.execute("DELETE FROM state_meta WHERE key = 'fts_storage_version'")
+ db._conn.commit()
# Plant markers without going through demote.
db.set_meta("fts_rebuild_high_water", "1")
db.set_meta("fts_rebuild_progress", "0")