Skip to content

The agent that grows with you

fix(state): leave an unrecognised base index alone in the open-time realign

yoniebansb3bfe6bf81f9
diff --git a/hermes_state_fts.py b/hermes_state_fts.py
index 0829be597d7..7193e1e3eba 100644
--- a/hermes_state_fts.py
+++ b/hermes_state_fts.py
@@ -132,9 +132,11 @@ _FTS_DDL_TABLES = ("messages_fts", "messages_fts_trigram")
def _stored_form(sql: str) -> str:
- """``sqlite_master.sql`` as compared here: whitespace collapsed and the object name unquoted (an ALTER TABLE
- RENAME round trip stores it as ``"name"`` without changing the layout)."""
- return re.sub(r'^CREATE (VIRTUAL TABLE|VIEW) "([^"]+)"', r"CREATE \1 \2", " ".join((sql or "").split()))
+ """``sqlite_master.sql`` as compared here: whitespace collapsed, none kept around parentheses and commas (the
+ same definition written with different spacing is the same layout), and the object name unquoted (an ALTER
+ TABLE RENAME round trip stores it as ``"name"`` without changing the layout)."""
+ collapsed = re.sub(r"\s*([(),])\s*", r"\1", " ".join((sql or "").split()))
+ return re.sub(r'^CREATE (VIRTUAL TABLE|VIEW) "([^"]+)"', r"CREATE \1 \2", collapsed)
def _expected_stored_sql(ddl: str, kind: str, name: str) -> str:
@@ -156,21 +158,21 @@ def _expected_vtable_sql(ddl: str, table: str) -> str:
# previous form recognisable as old.
FTS_VTABLE_LAYOUTS: dict[str, tuple[str, ...]] = {
"messages_fts": (
- "CREATE VIRTUAL TABLE messages_fts USING fts5( content, content=messages, content_rowid=id )",
- "CREATE VIRTUAL TABLE messages_fts USING fts5( content )",
- "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages', "
- "content_rowid='id' )",
- "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages_fts_src', "
- "content_rowid='id' )",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content,content=messages,content_rowid=id)",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content)",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content,tool_name,tool_calls,content='messages',"
+ "content_rowid='id')",
+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content,tool_name,tool_calls,"
+ "content='messages_fts_src',content_rowid='id')",
),
"messages_fts_trigram": (
- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, content=messages, content_rowid=id, "
- "tokenize='trigram' )",
- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tokenize='trigram' )",
- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, tool_calls, "
- "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",
- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, "
- "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,content=messages,content_rowid=id,"
+ "tokenize='trigram')",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,tokenize='trigram')",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,tool_name,tool_calls,"
+ "content='messages_fts_trigram_src',content_rowid='id',tokenize='trigram')",
+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,tool_name,"
+ "content='messages_fts_trigram_src',content_rowid='id',tokenize='trigram')",
),
}
@@ -191,6 +193,12 @@ def _is_older_layout(table: str, stored: str) -> bool:
return "tool_calls" in stored.lower()
+def base_index_recognised(cursor: sqlite3.Cursor) -> bool:
+ """``messages_fts`` holds a definition this code or an earlier release created; the open-time realign
+ replaces nothing else."""
+ return _stored_vtables(cursor).get("messages_fts") in FTS_VTABLE_LAYOUTS["messages_fts"]
+
+
def base_index_realigned_on_open(stored: str) -> bool:
"""External-content base index not reading ``messages_fts_src``: the next read-write open rebuilds it from
the current DDL (``_migrate_misaligned_fts_source``), so it is never opt-in work."""
diff --git a/hermes_state_schema.py b/hermes_state_schema.py
index afb64314c65..40345d86b8d 100644
--- a/hermes_state_schema.py
+++ b/hermes_state_schema.py
@@ -26,7 +26,8 @@ from hermes_state_common import (
LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,
SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,
)
-from hermes_state_fts import _drop_orphan_fts_shadow_tables, base_source_view_is_stale, fts_layout_current
+from hermes_state_fts import (_drop_orphan_fts_shadow_tables, base_index_recognised, base_source_view_is_stale,
+ fts_layout_current)
from hermes_state_holders import _read_proc_argv
from hermes_state_search import _delete_meta, _meta_row
from hermes_state_errors import is_sqlite_lock_error
@@ -355,8 +356,9 @@ class SessionSchemaMixin:
in-place continuity is not achievable — the ONLY valid transition is a full rebuild
from the view, under the shared cross-process rebuild admission. Legacy inline DBs
skip this entirely (their index is self-contained; they still take the DDL on the
- optimize path)."""
- if legacy or not self._sqlite_table_exists(cursor, "messages_fts"):
+ optimize path), and so does a definition no release of this code created (a newer
+ release's, after a downgrade)."""
+ if legacy or not base_index_recognised(cursor):
return
stale_view = base_source_view_is_stale(cursor)
if not stale_view and not self._fts_index_is_misaligned_source(cursor):
diff --git a/tests/hermes_state/test_fts_layout_drift.py b/tests/hermes_state/test_fts_layout_drift.py
index b368f9bc3fe..bf2167b2207 100644
--- a/tests/hermes_state/test_fts_layout_drift.py
+++ b/tests/hermes_state/test_fts_layout_drift.py
@@ -234,6 +234,23 @@ def test_the_layout_registry_ends_with_the_shipping_ddl():
LEGACY_FTS_TRIGRAM_SQL, "messages_fts_trigram") in layouts["messages_fts_trigram"][:-1]
+def _install_newer_base_layout(db: SessionDB) -> str:
+ """A base-index definition this code does not recognise (a newer release's), stamped with a newer version."""
+ from hermes_state_common import FTS_SQL, FTS_STORAGE_VERSION
+
+ sql = db._conn.execute("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").fetchone()[0]
+ db._conn.executescript(
+ "DROP TRIGGER messages_fts_insert; DROP TRIGGER messages_fts_delete; DROP TRIGGER messages_fts_update;"
+ "DROP TABLE messages_fts;")
+ newer = sql.replace("content_rowid='id'", "content_rowid='id', tokenize='porter unicode61'")
+ db._conn.execute(newer)
+ db._conn.executescript(FTS_SQL)
+ db._conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
+ db._conn.execute("UPDATE state_meta SET value = ? WHERE key = 'fts_storage_version'", (str(FTS_STORAGE_VERSION + 1),))
+ db._conn.commit()
+ return newer
+
+
def _forbid_demote(db: SessionDB, monkeypatch) -> None:
def _demote():
pytest.fail("optimize-storage demoted an index it does not recognise as an old layout")
@@ -242,20 +259,12 @@ def _forbid_demote(db: SessionDB, monkeypatch) -> None:
def test_a_layout_from_a_newer_release_is_left_alone(tmp_path, monkeypatch):
"""Downgrade: an unrecognised (newer) definition is neither rebuilt nor re-stamped as this release's."""
- from hermes_state_common import FTS_SQL, FTS_STORAGE_VERSION
+ from hermes_state_common import FTS_STORAGE_VERSION
db_path = tmp_path / "state.db"
db = SessionDB(db_path=db_path)
_seed(db)
- sql = db._conn.execute("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").fetchone()[0]
- db._conn.executescript(
- "DROP TRIGGER messages_fts_insert; DROP TRIGGER messages_fts_delete; DROP TRIGGER messages_fts_update;"
- "DROP TABLE messages_fts;")
- db._conn.execute(sql.replace("content_rowid='id'", "content_rowid='id', tokenize='porter unicode61'"))
- db._conn.executescript(FTS_SQL)
- db._conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
- db._conn.execute("UPDATE state_meta SET value = ? WHERE key = 'fts_storage_version'", (str(FTS_STORAGE_VERSION + 1),))
- db._conn.commit()
+ _install_newer_base_layout(db)
db.close()
db = SessionDB(db_path=db_path)
@@ -372,6 +381,28 @@ def test_the_base_realign_finishes_a_pending_trigram_backfill(tmp_path):
db.close()
+def test_a_stale_source_view_does_not_replace_a_newer_base_index(tmp_path):
+ """Downgrade with a differing source view: the open-time repair leaves an unrecognised base index alone."""
+ from hermes_state_common import FTS_STORAGE_VERSION
+
+ db_path = tmp_path / "state.db"
+ db = SessionDB(db_path=db_path)
+ _seed(db)
+ newer = _install_newer_base_layout(db)
+ db._conn.execute("DROP VIEW messages_fts_src")
+ db._conn.execute("CREATE VIEW messages_fts_src AS SELECT id, content, tool_name, tool_calls FROM messages")
+ db._conn.commit()
+ db.close()
+
+ SessionDB(db_path=db_path).close()
+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+ try:
+ assert conn.execute("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").fetchone()[0] == newer
+ finally:
+ conn.close()
+ assert _stamp(db_path) == str(FTS_STORAGE_VERSION + 1)
+
+
def test_doctor_offers_the_resume_after_backfill_before_teardown(tmp_path, monkeypatch):
"""optimize-storage interrupted after backfill but before trash teardown: doctor agrees with the update notice."""
from hermes_cli import doctor_state