The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(state): leave an unrecognised base index alone in the open-time realign
yoniebansb3bfe6bf81f9
diff --git a/hermes_state_fts.py b/hermes_state_fts.pyindex 0829be597d7..7193e1e3eba 100644--- a/hermes_state_fts.py+++ b/hermes_state_fts.py@@ -132,9 +132,11 @@ _FTS_DDL_TABLES = ("messages_fts", "messages_fts_trigram")def _stored_form(sql: str) -> str:- """``sqlite_master.sql`` as compared here: whitespace collapsed and the object name unquoted (an ALTER TABLE- RENAME round trip stores it as ``"name"`` without changing the layout)."""- return re.sub(r'^CREATE (VIRTUAL TABLE|VIEW) "([^"]+)"', r"CREATE \1 \2", " ".join((sql or "").split()))+ """``sqlite_master.sql`` as compared here: whitespace collapsed, none kept around parentheses and commas (the+ same definition written with different spacing is the same layout), and the object name unquoted (an ALTER+ TABLE RENAME round trip stores it as ``"name"`` without changing the layout)."""+ collapsed = re.sub(r"\s*([(),])\s*", r"\1", " ".join((sql or "").split()))+ return re.sub(r'^CREATE (VIRTUAL TABLE|VIEW) "([^"]+)"', r"CREATE \1 \2", collapsed)def _expected_stored_sql(ddl: str, kind: str, name: str) -> str:@@ -156,21 +158,21 @@ def _expected_vtable_sql(ddl: str, table: str) -> str:# previous form recognisable as old.FTS_VTABLE_LAYOUTS: dict[str, tuple[str, ...]] = {"messages_fts": (- "CREATE VIRTUAL TABLE messages_fts USING fts5( content, content=messages, content_rowid=id )",- "CREATE VIRTUAL TABLE messages_fts USING fts5( content )",- "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages', "- "content_rowid='id' )",- "CREATE VIRTUAL TABLE messages_fts USING fts5( content, tool_name, tool_calls, content='messages_fts_src', "- "content_rowid='id' )",+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content,content=messages,content_rowid=id)",+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content)",+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content,tool_name,tool_calls,content='messages',"+ "content_rowid='id')",+ "CREATE VIRTUAL TABLE messages_fts USING fts5(content,tool_name,tool_calls,"+ "content='messages_fts_src',content_rowid='id')",),"messages_fts_trigram": (- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, content=messages, content_rowid=id, "- "tokenize='trigram' )",- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tokenize='trigram' )",- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, tool_calls, "- "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",- "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( content, tool_name, "- "content='messages_fts_trigram_src', content_rowid='id', tokenize='trigram' )",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,content=messages,content_rowid=id,"+ "tokenize='trigram')",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,tokenize='trigram')",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,tool_name,tool_calls,"+ "content='messages_fts_trigram_src',content_rowid='id',tokenize='trigram')",+ "CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content,tool_name,"+ "content='messages_fts_trigram_src',content_rowid='id',tokenize='trigram')",),}@@ -191,6 +193,12 @@ def _is_older_layout(table: str, stored: str) -> bool:return "tool_calls" in stored.lower()+def base_index_recognised(cursor: sqlite3.Cursor) -> bool:+ """``messages_fts`` holds a definition this code or an earlier release created; the open-time realign+ replaces nothing else."""+ return _stored_vtables(cursor).get("messages_fts") in FTS_VTABLE_LAYOUTS["messages_fts"]++def base_index_realigned_on_open(stored: str) -> bool:"""External-content base index not reading ``messages_fts_src``: the next read-write open rebuilds it fromthe current DDL (``_migrate_misaligned_fts_source``), so it is never opt-in work."""diff --git a/hermes_state_schema.py b/hermes_state_schema.pyindex afb64314c65..40345d86b8d 100644--- a/hermes_state_schema.py+++ b/hermes_state_schema.py@@ -26,7 +26,8 @@ from hermes_state_common import (LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,)-from hermes_state_fts import _drop_orphan_fts_shadow_tables, base_source_view_is_stale, fts_layout_current+from hermes_state_fts import (_drop_orphan_fts_shadow_tables, base_index_recognised, base_source_view_is_stale,+ fts_layout_current)from hermes_state_holders import _read_proc_argvfrom hermes_state_search import _delete_meta, _meta_rowfrom hermes_state_errors import is_sqlite_lock_error@@ -355,8 +356,9 @@ class SessionSchemaMixin:in-place continuity is not achievable — the ONLY valid transition is a full rebuildfrom the view, under the shared cross-process rebuild admission. Legacy inline DBsskip this entirely (their index is self-contained; they still take the DDL on the- optimize path)."""- if legacy or not self._sqlite_table_exists(cursor, "messages_fts"):+ optimize path), and so does a definition no release of this code created (a newer+ release's, after a downgrade)."""+ if legacy or not base_index_recognised(cursor):returnstale_view = base_source_view_is_stale(cursor)if not stale_view and not self._fts_index_is_misaligned_source(cursor):diff --git a/tests/hermes_state/test_fts_layout_drift.py b/tests/hermes_state/test_fts_layout_drift.pyindex b368f9bc3fe..bf2167b2207 100644--- a/tests/hermes_state/test_fts_layout_drift.py+++ b/tests/hermes_state/test_fts_layout_drift.py@@ -234,6 +234,23 @@ def test_the_layout_registry_ends_with_the_shipping_ddl():LEGACY_FTS_TRIGRAM_SQL, "messages_fts_trigram") in layouts["messages_fts_trigram"][:-1]+def _install_newer_base_layout(db: SessionDB) -> str:+ """A base-index definition this code does not recognise (a newer release's), stamped with a newer version."""+ from hermes_state_common import FTS_SQL, FTS_STORAGE_VERSION++ sql = db._conn.execute("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").fetchone()[0]+ db._conn.executescript(+ "DROP TRIGGER messages_fts_insert; DROP TRIGGER messages_fts_delete; DROP TRIGGER messages_fts_update;"+ "DROP TABLE messages_fts;")+ newer = sql.replace("content_rowid='id'", "content_rowid='id', tokenize='porter unicode61'")+ db._conn.execute(newer)+ db._conn.executescript(FTS_SQL)+ db._conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")+ db._conn.execute("UPDATE state_meta SET value = ? WHERE key = 'fts_storage_version'", (str(FTS_STORAGE_VERSION + 1),))+ db._conn.commit()+ return newer++def _forbid_demote(db: SessionDB, monkeypatch) -> None:def _demote():pytest.fail("optimize-storage demoted an index it does not recognise as an old layout")@@ -242,20 +259,12 @@ def _forbid_demote(db: SessionDB, monkeypatch) -> None:def test_a_layout_from_a_newer_release_is_left_alone(tmp_path, monkeypatch):"""Downgrade: an unrecognised (newer) definition is neither rebuilt nor re-stamped as this release's."""- from hermes_state_common import FTS_SQL, FTS_STORAGE_VERSION+ from hermes_state_common import FTS_STORAGE_VERSIONdb_path = tmp_path / "state.db"db = SessionDB(db_path=db_path)_seed(db)- sql = db._conn.execute("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").fetchone()[0]- db._conn.executescript(- "DROP TRIGGER messages_fts_insert; DROP TRIGGER messages_fts_delete; DROP TRIGGER messages_fts_update;"- "DROP TABLE messages_fts;")- db._conn.execute(sql.replace("content_rowid='id'", "content_rowid='id', tokenize='porter unicode61'"))- db._conn.executescript(FTS_SQL)- db._conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")- db._conn.execute("UPDATE state_meta SET value = ? WHERE key = 'fts_storage_version'", (str(FTS_STORAGE_VERSION + 1),))- db._conn.commit()+ _install_newer_base_layout(db)db.close()db = SessionDB(db_path=db_path)@@ -372,6 +381,28 @@ def test_the_base_realign_finishes_a_pending_trigram_backfill(tmp_path):db.close()+def test_a_stale_source_view_does_not_replace_a_newer_base_index(tmp_path):+ """Downgrade with a differing source view: the open-time repair leaves an unrecognised base index alone."""+ from hermes_state_common import FTS_STORAGE_VERSION++ db_path = tmp_path / "state.db"+ db = SessionDB(db_path=db_path)+ _seed(db)+ newer = _install_newer_base_layout(db)+ db._conn.execute("DROP VIEW messages_fts_src")+ db._conn.execute("CREATE VIEW messages_fts_src AS SELECT id, content, tool_name, tool_calls FROM messages")+ db._conn.commit()+ db.close()++ SessionDB(db_path=db_path).close()+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)+ try:+ assert conn.execute("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").fetchone()[0] == newer+ finally:+ conn.close()+ assert _stamp(db_path) == str(FTS_STORAGE_VERSION + 1)++def test_doctor_offers_the_resume_after_backfill_before_teardown(tmp_path, monkeypatch):"""optimize-storage interrupted after backfill but before trash teardown: doctor agrees with the update notice."""from hermes_cli import doctor_state