Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(search): accent-insensitive FTS (remove_diacritics)

Djalma Júnioreee7d629f622
diff --git a/crates/ai-memory-store/migrations/V13__fts_remove_diacritics.sql b/crates/ai-memory-store/migrations/V13__fts_remove_diacritics.sql
new file mode 100644
index 00000000..3e14b8e0
--- /dev/null
+++ b/crates/ai-memory-store/migrations/V13__fts_remove_diacritics.sql
@@ -0,0 +1,70 @@
+-- Accent-insensitive full-text search (Portuguese-friendly).
+--
+-- The FTS5 tables were created with `unicode61 tokenchars '/_-'`, which keeps
+-- diacritics: a query for "descricao" never matches stored "descrição". The
+-- tokenizer is fixed at CREATE time, so the only way to change it is to drop
+-- and recreate the FTS table (+ its sync triggers) and rebuild from the
+-- content table. `remove_diacritics 2` folds diacritics across the full
+-- Unicode range, so "descricao"/"descrição", "sessao"/"sessão", etc. unify.
+--
+-- Both FTS tables are contentless (`content='pages'` / `content='observations'`),
+-- so the source rows are untouched; only the derived index is rebuilt.
+
+-- ── pages_fts ────────────────────────────────────────────────────────────
+DROP TRIGGER pages_fts_ai;
+DROP TRIGGER pages_fts_ad;
+DROP TRIGGER pages_fts_au;
+DROP TABLE pages_fts;
+
+CREATE VIRTUAL TABLE pages_fts USING fts5(
+ title, body,
+ content='pages',
+ content_rowid='rowid',
+ tokenize="unicode61 remove_diacritics 2 tokenchars '/_-'"
+);
+INSERT INTO pages_fts(pages_fts) VALUES('rebuild');
+
+CREATE TRIGGER pages_fts_ai AFTER INSERT ON pages BEGIN
+ INSERT INTO pages_fts(rowid, title, body)
+ VALUES (new.rowid, new.title, new.body);
+END;
+CREATE TRIGGER pages_fts_ad AFTER DELETE ON pages BEGIN
+ INSERT INTO pages_fts(pages_fts, rowid, title, body)
+ VALUES ('delete', old.rowid, old.title, old.body);
+END;
+-- Matches the V08 narrowing: only re-index on title/body updates.
+CREATE TRIGGER pages_fts_au AFTER UPDATE OF title, body ON pages BEGIN
+ INSERT INTO pages_fts(pages_fts, rowid, title, body)
+ VALUES ('delete', old.rowid, old.title, old.body);
+ INSERT INTO pages_fts(rowid, title, body)
+ VALUES (new.rowid, new.title, new.body);
+END;
+
+-- ── observations_fts ──────────────────────────────────────────────────────
+DROP TRIGGER observations_fts_ai;
+DROP TRIGGER observations_fts_ad;
+DROP TRIGGER observations_fts_au;
+DROP TABLE observations_fts;
+
+CREATE VIRTUAL TABLE observations_fts USING fts5(
+ title, body,
+ content='observations',
+ content_rowid='rowid',
+ tokenize="unicode61 remove_diacritics 2 tokenchars '/_-'"
+);
+INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
+
+CREATE TRIGGER observations_fts_ai AFTER INSERT ON observations BEGIN
+ INSERT INTO observations_fts(rowid, title, body)
+ VALUES (new.rowid, new.title, new.body);
+END;
+CREATE TRIGGER observations_fts_ad AFTER DELETE ON observations BEGIN
+ INSERT INTO observations_fts(observations_fts, rowid, title, body)
+ VALUES ('delete', old.rowid, old.title, old.body);
+END;
+CREATE TRIGGER observations_fts_au AFTER UPDATE ON observations BEGIN
+ INSERT INTO observations_fts(observations_fts, rowid, title, body)
+ VALUES ('delete', old.rowid, old.title, old.body);
+ INSERT INTO observations_fts(rowid, title, body)
+ VALUES (new.rowid, new.title, new.body);
+END;
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index a003d8a5..9a9359f4 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -414,6 +414,43 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn search_is_accent_insensitive() {
+ // V13: an accent-free query matches accented stored text (PT-friendly).
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(sample_page(
+ ws,
+ proj,
+ "notes/decisao.md",
+ "a descrição da sessão e a consolidação dos commits",
+ ))
+ .await
+ .unwrap();
+
+ let hits = store
+ .reader
+ .search_pages("descricao sessao".into(), 10)
+ .await
+ .unwrap();
+ assert!(
+ !hits.is_empty(),
+ "accent-free query must match accented stored text"
+ );
+ }
+
#[tokio::test]
async fn search_boolean_or_still_works() {
let tmp = TempDir::new().unwrap();