Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(store): index page path/slug in FTS so filename searches hit
Djalma Júnior5ce34537fc73
diff --git a/crates/ai-memory-store/migrations/V17__pages_fts_index_path.sql b/crates/ai-memory-store/migrations/V17__pages_fts_index_path.sqlnew file mode 100644index 00000000..74a2f98c--- /dev/null+++ b/crates/ai-memory-store/migrations/V17__pages_fts_index_path.sql@@ -0,0 +1,57 @@+-- Index the page PATH (slug/filename) in the FTS table so a search for a+-- distinctive slug like `followup-bulk-rename-runbook-titles` finds the page+-- even when the slug never appears in the prose body or title.+--+-- The FTS tokenizer keeps `/`, `_`, `-` as token characters (so identifiers+-- stay whole), which means a raw path `notes/foo-bar.md` tokenizes as the+-- single glued token `notes/foo-bar` plus `md`. We index a NORMALISED copy of+-- the path in BOTH forms so either query style hits:+-- * segments — `/` and `.` → space, KEEPING `-`/`_` (so the whole hyphenated+-- slug stays one token: `"foo-bar"` matches);+-- * words — also split `-`/`_` (so `bar` alone matches).+-- `notes/foo-bar.md` → `notes foo-bar md notes foo bar md`. That text lives in+-- a real `pages.path_search` column so the FTS stays content-backed (external+-- content; no body duplication) and the writer keeps it in sync. This SQL MUST+-- match `ops::path_search_text` byte-for-byte.++ALTER TABLE pages ADD COLUMN path_search TEXT NOT NULL DEFAULT '';++UPDATE pages+ SET path_search =+ replace(replace(path, '/', ' '), '.', ' ')+ || ' '+ || replace(replace(replace(replace(path, '/', ' '), '.', ' '), '-', ' '), '_', ' ');++-- Recreate the FTS table + triggers with the extra column. The old triggers+-- reference the 2-column shape, so they must be dropped first.+DROP TRIGGER IF EXISTS pages_fts_ai;+DROP TRIGGER IF EXISTS pages_fts_ad;+DROP TRIGGER IF EXISTS pages_fts_au;+DROP TABLE IF EXISTS pages_fts;++CREATE VIRTUAL TABLE pages_fts USING fts5(+ title, body, path_search,+ content='pages',+ content_rowid='rowid',+ tokenize="unicode61 tokenchars '/_-'"+);++CREATE TRIGGER pages_fts_ai AFTER INSERT ON pages BEGIN+ INSERT INTO pages_fts(rowid, title, body, path_search)+ VALUES (new.rowid, new.title, new.body, new.path_search);+END;++CREATE TRIGGER pages_fts_ad AFTER DELETE ON pages BEGIN+ INSERT INTO pages_fts(pages_fts, rowid, title, body, path_search)+ VALUES ('delete', old.rowid, old.title, old.body, old.path_search);+END;++CREATE TRIGGER pages_fts_au AFTER UPDATE ON pages BEGIN+ INSERT INTO pages_fts(pages_fts, rowid, title, body, path_search)+ VALUES ('delete', old.rowid, old.title, old.body, old.path_search);+ INSERT INTO pages_fts(rowid, title, body, path_search)+ VALUES (new.rowid, new.title, new.body, new.path_search);+END;++-- Repopulate the FTS index from the (now path_search-bearing) content table.+INSERT INTO pages_fts(pages_fts) VALUES('rebuild');diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 70258c1e..3e682a8c 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -169,11 +169,30 @@ struct ExistingPageVersion {pinned: i64,}+/// Normalise a page path into FTS-friendly search text, indexing BOTH forms+/// so either a whole-slug or a single-word query hits:+/// - segments: `/` and `.` → space, KEEPING `-`/`_` (FTS token chars) so the+/// full hyphenated slug stays one token (`foo-bar` matches a `"foo-bar"`+/// query);+/// - words: also split `-`/`_` so each word is its own token (`bar` matches).+///+/// `notes/foo-bar.md` → `notes foo-bar md notes foo bar md`.+///+/// MUST stay byte-identical to the backfill expression in migration V17 so+/// the `rebuild` and live-write paths index the same text (matching bm25+/// term frequencies, not just the same match set).+pub(crate) fn path_search_text(path: &str) -> String {+ let segments = path.replace(['/', '.'], " ");+ let words = segments.replace(['-', '_'], " ");+ format!("{segments} {words}")+}+fn upsert_page_in_tx(tx: &rusqlite::Transaction<'_>,page: &NewPage,now: i64,) -> StoreResult<PageId> {+ let path_search = path_search_text(page.path.as_str());let body_sha256: [u8; 32] = {let mut hasher = Sha256::new();hasher.update(page.body.as_bytes());@@ -220,15 +239,16 @@ fn upsert_page_in_tx()?;tx.execute("INSERT INTO pages \- (id, workspace_id, project_id, path, title, tier, body, body_sha256, \+ (id, workspace_id, project_id, path, path_search, title, tier, body, body_sha256, \frontmatter_json, is_latest, supersedes, pinned, author_id, \created_at, updated_at) \- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1, ?10, ?11, ?12, ?13, ?13)",+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 1, ?11, ?12, ?13, ?14, ?14)",params![new_id.as_bytes(),page.workspace_id.as_bytes(),page.project_id.as_bytes(),page.path.as_str(),+ path_search,page.title,tier_str,page.body,@@ -258,14 +278,15 @@ fn upsert_page_in_tx(let new_id = PageId::new();tx.execute("INSERT INTO pages \- (id, workspace_id, project_id, path, title, tier, body, body_sha256, \+ (id, workspace_id, project_id, path, path_search, title, tier, body, body_sha256, \frontmatter_json, is_latest, pinned, author_id, created_at, updated_at) \- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1, ?10, ?11, ?12, ?12)",+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 1, ?11, ?12, ?13, ?13)",params![new_id.as_bytes(),page.workspace_id.as_bytes(),page.project_id.as_bytes(),page.path.as_str(),+ path_search,page.title,tier_str,page.body,@@ -1686,4 +1707,62 @@ mod tests {"explicit project purge must not touch siblings");}++ #[test]+ fn path_search_text_indexes_slug_and_words() {+ // Both forms: hyphenated slug kept whole, plus split into words.+ assert_eq!(+ path_search_text("notes/foo-bar.md"),+ "notes foo-bar md notes foo bar md"+ );+ assert_eq!(path_search_text("a/b_c.md"), "a b_c md a b c md");+ }++ /// A page is findable by its PATH slug even when the slug appears in+ /// neither the title nor the body — the V17 `path_search` FTS column.+ #[test]+ fn fts_matches_page_by_path_slug_not_in_body() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ // Title + body deliberately avoid the slug words.+ let mut p = page(+ ws,+ proj,+ "notes/followup-bulk-rename-runbook-titles.md",+ "totally unrelated prose about elephants",+ );+ p.title = "Elephants".into();+ upsert_page(&mut conn, &p).unwrap();++ // The slug, as a quoted single token (how prepare_fts5_query renders a+ // hyphenated term), matches via the path_search column.+ let n: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM pages_fts \+ WHERE pages_fts MATCH ?1",+ params!["\"followup-bulk-rename-runbook-titles\""],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(n, 1, "slug in path must be searchable");++ // A distinct path segment is independently searchable too.+ let seg: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM pages_fts WHERE pages_fts MATCH 'runbook'",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(seg, 1, "path segment token must match");++ // Body words still match (regression: body stays indexed at col 1).+ let body_hit: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM pages_fts WHERE pages_fts MATCH 'elephants'",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(body_hit, 1, "body must remain searchable");+ }}