Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(search): accent-insensitive FTS (remove_diacritics)
Djalma Júnioreee7d629f622
diff --git a/crates/ai-memory-store/migrations/V13__fts_remove_diacritics.sql b/crates/ai-memory-store/migrations/V13__fts_remove_diacritics.sqlnew file mode 100644index 00000000..3e14b8e0--- /dev/null+++ b/crates/ai-memory-store/migrations/V13__fts_remove_diacritics.sql@@ -0,0 +1,70 @@+-- Accent-insensitive full-text search (Portuguese-friendly).+--+-- The FTS5 tables were created with `unicode61 tokenchars '/_-'`, which keeps+-- diacritics: a query for "descricao" never matches stored "descrição". The+-- tokenizer is fixed at CREATE time, so the only way to change it is to drop+-- and recreate the FTS table (+ its sync triggers) and rebuild from the+-- content table. `remove_diacritics 2` folds diacritics across the full+-- Unicode range, so "descricao"/"descrição", "sessao"/"sessão", etc. unify.+--+-- Both FTS tables are contentless (`content='pages'` / `content='observations'`),+-- so the source rows are untouched; only the derived index is rebuilt.++-- ── pages_fts ────────────────────────────────────────────────────────────+DROP TRIGGER pages_fts_ai;+DROP TRIGGER pages_fts_ad;+DROP TRIGGER pages_fts_au;+DROP TABLE pages_fts;++CREATE VIRTUAL TABLE pages_fts USING fts5(+ title, body,+ content='pages',+ content_rowid='rowid',+ tokenize="unicode61 remove_diacritics 2 tokenchars '/_-'"+);+INSERT INTO pages_fts(pages_fts) VALUES('rebuild');++CREATE TRIGGER pages_fts_ai AFTER INSERT ON pages BEGIN+ INSERT INTO pages_fts(rowid, title, body)+ VALUES (new.rowid, new.title, new.body);+END;+CREATE TRIGGER pages_fts_ad AFTER DELETE ON pages BEGIN+ INSERT INTO pages_fts(pages_fts, rowid, title, body)+ VALUES ('delete', old.rowid, old.title, old.body);+END;+-- Matches the V08 narrowing: only re-index on title/body updates.+CREATE TRIGGER pages_fts_au AFTER UPDATE OF title, body ON pages BEGIN+ INSERT INTO pages_fts(pages_fts, rowid, title, body)+ VALUES ('delete', old.rowid, old.title, old.body);+ INSERT INTO pages_fts(rowid, title, body)+ VALUES (new.rowid, new.title, new.body);+END;++-- ── observations_fts ──────────────────────────────────────────────────────+DROP TRIGGER observations_fts_ai;+DROP TRIGGER observations_fts_ad;+DROP TRIGGER observations_fts_au;+DROP TABLE observations_fts;++CREATE VIRTUAL TABLE observations_fts USING fts5(+ title, body,+ content='observations',+ content_rowid='rowid',+ tokenize="unicode61 remove_diacritics 2 tokenchars '/_-'"+);+INSERT INTO observations_fts(observations_fts) VALUES('rebuild');++CREATE TRIGGER observations_fts_ai AFTER INSERT ON observations BEGIN+ INSERT INTO observations_fts(rowid, title, body)+ VALUES (new.rowid, new.title, new.body);+END;+CREATE TRIGGER observations_fts_ad AFTER DELETE ON observations BEGIN+ INSERT INTO observations_fts(observations_fts, rowid, title, body)+ VALUES ('delete', old.rowid, old.title, old.body);+END;+CREATE TRIGGER observations_fts_au AFTER UPDATE ON observations BEGIN+ INSERT INTO observations_fts(observations_fts, rowid, title, body)+ VALUES ('delete', old.rowid, old.title, old.body);+ INSERT INTO observations_fts(rowid, title, body)+ VALUES (new.rowid, new.title, new.body);+END;diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex a003d8a5..9a9359f4 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -414,6 +414,43 @@ mod tests {);}+ #[tokio::test]+ async fn search_is_accent_insensitive() {+ // V13: an accent-free query matches accented stored text (PT-friendly).+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();+ store+ .writer+ .upsert_page(sample_page(+ ws,+ proj,+ "notes/decisao.md",+ "a descrição da sessão e a consolidação dos commits",+ ))+ .await+ .unwrap();++ let hits = store+ .reader+ .search_pages("descricao sessao".into(), 10)+ .await+ .unwrap();+ assert!(+ !hits.is_empty(),+ "accent-free query must match accented stored text"+ );+ }+#[tokio::test]async fn search_boolean_or_still_works() {let tmp = TempDir::new().unwrap();