Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(store): index page path/slug in FTS so filename searches hit

Djalma Júnior5ce34537fc73
diff --git a/crates/ai-memory-store/migrations/V17__pages_fts_index_path.sql b/crates/ai-memory-store/migrations/V17__pages_fts_index_path.sql
new file mode 100644
index 00000000..74a2f98c
--- /dev/null
+++ b/crates/ai-memory-store/migrations/V17__pages_fts_index_path.sql
@@ -0,0 +1,57 @@
+-- Index the page PATH (slug/filename) in the FTS table so a search for a
+-- distinctive slug like `followup-bulk-rename-runbook-titles` finds the page
+-- even when the slug never appears in the prose body or title.
+--
+-- The FTS tokenizer keeps `/`, `_`, `-` as token characters (so identifiers
+-- stay whole), which means a raw path `notes/foo-bar.md` tokenizes as the
+-- single glued token `notes/foo-bar` plus `md`. We index a NORMALISED copy of
+-- the path in BOTH forms so either query style hits:
+-- * segments — `/` and `.` → space, KEEPING `-`/`_` (so the whole hyphenated
+-- slug stays one token: `"foo-bar"` matches);
+-- * words — also split `-`/`_` (so `bar` alone matches).
+-- `notes/foo-bar.md` → `notes foo-bar md notes foo bar md`. That text lives in
+-- a real `pages.path_search` column so the FTS stays content-backed (external
+-- content; no body duplication) and the writer keeps it in sync. This SQL MUST
+-- match `ops::path_search_text` byte-for-byte.
+
+ALTER TABLE pages ADD COLUMN path_search TEXT NOT NULL DEFAULT '';
+
+UPDATE pages
+ SET path_search =
+ replace(replace(path, '/', ' '), '.', ' ')
+ || ' '
+ || replace(replace(replace(replace(path, '/', ' '), '.', ' '), '-', ' '), '_', ' ');
+
+-- Recreate the FTS table + triggers with the extra column. The old triggers
+-- reference the 2-column shape, so they must be dropped first.
+DROP TRIGGER IF EXISTS pages_fts_ai;
+DROP TRIGGER IF EXISTS pages_fts_ad;
+DROP TRIGGER IF EXISTS pages_fts_au;
+DROP TABLE IF EXISTS pages_fts;
+
+CREATE VIRTUAL TABLE pages_fts USING fts5(
+ title, body, path_search,
+ content='pages',
+ content_rowid='rowid',
+ tokenize="unicode61 tokenchars '/_-'"
+);
+
+CREATE TRIGGER pages_fts_ai AFTER INSERT ON pages BEGIN
+ INSERT INTO pages_fts(rowid, title, body, path_search)
+ VALUES (new.rowid, new.title, new.body, new.path_search);
+END;
+
+CREATE TRIGGER pages_fts_ad AFTER DELETE ON pages BEGIN
+ INSERT INTO pages_fts(pages_fts, rowid, title, body, path_search)
+ VALUES ('delete', old.rowid, old.title, old.body, old.path_search);
+END;
+
+CREATE TRIGGER pages_fts_au AFTER UPDATE ON pages BEGIN
+ INSERT INTO pages_fts(pages_fts, rowid, title, body, path_search)
+ VALUES ('delete', old.rowid, old.title, old.body, old.path_search);
+ INSERT INTO pages_fts(rowid, title, body, path_search)
+ VALUES (new.rowid, new.title, new.body, new.path_search);
+END;
+
+-- Repopulate the FTS index from the (now path_search-bearing) content table.
+INSERT INTO pages_fts(pages_fts) VALUES('rebuild');
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 70258c1e..3e682a8c 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -169,11 +169,30 @@ struct ExistingPageVersion {
pinned: i64,
}
+/// Normalise a page path into FTS-friendly search text, indexing BOTH forms
+/// so either a whole-slug or a single-word query hits:
+/// - segments: `/` and `.` → space, KEEPING `-`/`_` (FTS token chars) so the
+/// full hyphenated slug stays one token (`foo-bar` matches a `"foo-bar"`
+/// query);
+/// - words: also split `-`/`_` so each word is its own token (`bar` matches).
+///
+/// `notes/foo-bar.md` → `notes foo-bar md notes foo bar md`.
+///
+/// MUST stay byte-identical to the backfill expression in migration V17 so
+/// the `rebuild` and live-write paths index the same text (matching bm25
+/// term frequencies, not just the same match set).
+pub(crate) fn path_search_text(path: &str) -> String {
+ let segments = path.replace(['/', '.'], " ");
+ let words = segments.replace(['-', '_'], " ");
+ format!("{segments} {words}")
+}
+
fn upsert_page_in_tx(
tx: &rusqlite::Transaction<'_>,
page: &NewPage,
now: i64,
) -> StoreResult<PageId> {
+ let path_search = path_search_text(page.path.as_str());
let body_sha256: [u8; 32] = {
let mut hasher = Sha256::new();
hasher.update(page.body.as_bytes());
@@ -220,15 +239,16 @@ fn upsert_page_in_tx(
)?;
tx.execute(
"INSERT INTO pages \
- (id, workspace_id, project_id, path, title, tier, body, body_sha256, \
+ (id, workspace_id, project_id, path, path_search, title, tier, body, body_sha256, \
frontmatter_json, is_latest, supersedes, pinned, author_id, \
created_at, updated_at) \
- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1, ?10, ?11, ?12, ?13, ?13)",
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 1, ?11, ?12, ?13, ?14, ?14)",
params![
new_id.as_bytes(),
page.workspace_id.as_bytes(),
page.project_id.as_bytes(),
page.path.as_str(),
+ path_search,
page.title,
tier_str,
page.body,
@@ -258,14 +278,15 @@ fn upsert_page_in_tx(
let new_id = PageId::new();
tx.execute(
"INSERT INTO pages \
- (id, workspace_id, project_id, path, title, tier, body, body_sha256, \
+ (id, workspace_id, project_id, path, path_search, title, tier, body, body_sha256, \
frontmatter_json, is_latest, pinned, author_id, created_at, updated_at) \
- VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1, ?10, ?11, ?12, ?12)",
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 1, ?11, ?12, ?13, ?13)",
params![
new_id.as_bytes(),
page.workspace_id.as_bytes(),
page.project_id.as_bytes(),
page.path.as_str(),
+ path_search,
page.title,
tier_str,
page.body,
@@ -1686,4 +1707,62 @@ mod tests {
"explicit project purge must not touch siblings"
);
}
+
+ #[test]
+ fn path_search_text_indexes_slug_and_words() {
+ // Both forms: hyphenated slug kept whole, plus split into words.
+ assert_eq!(
+ path_search_text("notes/foo-bar.md"),
+ "notes foo-bar md notes foo bar md"
+ );
+ assert_eq!(path_search_text("a/b_c.md"), "a b_c md a b c md");
+ }
+
+ /// A page is findable by its PATH slug even when the slug appears in
+ /// neither the title nor the body — the V17 `path_search` FTS column.
+ #[test]
+ fn fts_matches_page_by_path_slug_not_in_body() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ // Title + body deliberately avoid the slug words.
+ let mut p = page(
+ ws,
+ proj,
+ "notes/followup-bulk-rename-runbook-titles.md",
+ "totally unrelated prose about elephants",
+ );
+ p.title = "Elephants".into();
+ upsert_page(&mut conn, &p).unwrap();
+
+ // The slug, as a quoted single token (how prepare_fts5_query renders a
+ // hyphenated term), matches via the path_search column.
+ let n: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM pages_fts \
+ WHERE pages_fts MATCH ?1",
+ params!["\"followup-bulk-rename-runbook-titles\""],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(n, 1, "slug in path must be searchable");
+
+ // A distinct path segment is independently searchable too.
+ let seg: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM pages_fts WHERE pages_fts MATCH 'runbook'",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(seg, 1, "path segment token must match");
+
+ // Body words still match (regression: body stays indexed at col 1).
+ let body_hit: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM pages_fts WHERE pages_fts MATCH 'elephants'",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(body_hit, 1, "body must remain searchable");
+ }
}