Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(fts): match hyphenated terms against both content and path index
Djalma Júnior5b45b687d3de
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rsindex 9200d8ca..9c787705 100644--- a/crates/ai-memory-store/src/fts_query.rs+++ b/crates/ai-memory-store/src/fts_query.rs@@ -76,9 +76,36 @@ fn should_quote_fts5_token(token: &str) -> bool {}fn quote_fts5_token(token: &str) -> String {- // FTS5 escapes `"` inside a quoted token by doubling it.- let escaped = token.replace('"', "\"\"");- format!("\"{escaped}\"")+ // FTS5 escapes `"` by doubling it. A token carrying a literal quote is an+ // explicit-phrase fragment — keep the simple escaped form (don't expand it).+ if token.contains('"') {+ return format!("\"{}\"", token.replace('"', "\"\""));+ }+ // Otherwise emit BOTH the whole token and a punctuation-stripped sub-token+ // phrase, OR'd, because the content tokenizer and the path index disagree+ // on punctuation:+ // tokenize = "unicode61 remove_diacritics 2 tokenchars '/_-'"+ // keeps `/ _ -` INSIDE tokens (so a body mention of `ai-memory` indexes as+ // the single token `ai-memory`), while `ops::path_search_text` pre-expands+ // `/ . - _` to spaces in the path index (so a path `ui-refresh-…` indexes+ // the sub-tokens `ui`, `refresh`, …). `.` is a separator either way.+ // Neither form alone matches both: `"ai-memory"` matches the content token+ // but not the split path index; `"ai memory"` matches the path but not the+ // content token. OR-ing the two makes a search for `ai-memory` / `ui-refresh`+ // hit whichever surface indexed it. (Quoting both also neutralises the+ // punctuation that would otherwise be FTS5 query grammar — the original+ // `current.md` → `syntax error` bug.) With no punctuation the two coincide+ // and we emit a single phrase.+ let split = token+ .chars()+ .map(|c| if c.is_ascii_punctuation() { ' ' } else { c })+ .collect::<String>();+ let split = split.split_whitespace().collect::<Vec<_>>().join(" ");+ if split.is_empty() || split == token {+ format!("\"{token}\"")+ } else {+ format!("(\"{token}\" OR \"{split}\")")+ }}#[cfg(test)]@@ -88,8 +115,11 @@ mod tests {#[test]fn colon_is_not_column_syntax() {// Bare multi-word → OR-joined (no explicit operator present).+ // `ai-memory` expands to BOTH the whole token (matches content) and+ // the sub-token phrase (matches the split path index) — see+ // `quote_fts5_token`.let q = prepare_fts5_query("pick: handoff ai-memory");- assert_eq!(q, "\"pick\" OR handoff OR \"ai-memory\"");+ assert_eq!(q, "\"pick\" OR handoff OR (\"ai-memory\" OR \"ai memory\")");}#[test]@@ -122,9 +152,37 @@ mod tests {/// `current` + `md` are adjacent in the indexed path).#[test]fn dotted_filename_token_is_quoted() {- assert_eq!(prepare_fts5_query("current.md"), "\"current.md\"");- assert_eq!(prepare_fts5_query("00-index.md"), "\"00-index.md\"");- assert_eq!(prepare_fts5_query("a/b/c.md"), "\"a/b/c.md\"");+ // Whole token OR sub-token phrase. The split form (`current md`)+ // matches the tokenised path; the whole form covers content tokens.+ assert_eq!(+ prepare_fts5_query("current.md"),+ "(\"current.md\" OR \"current md\")"+ );+ assert_eq!(+ prepare_fts5_query("00-index.md"),+ "(\"00-index.md\" OR \"00 index md\")"+ );+ assert_eq!(+ prepare_fts5_query("a/b/c.md"),+ "(\"a/b/c.md\" OR \"a b c md\")"+ );+ }++ /// Regression for the live-found bug: searching `ui-refresh` returned+ /// nothing even though `follow-ups/ui-refresh-scroll-restoration.md`+ /// exists. The old quoting produced `"ui-refresh"`, which FTS5 does NOT+ /// match against the indexed `ui refresh`; the sub-token phrase+ /// `"ui refresh"` does. See the real-FTS5 test in `ops.rs`.+ #[test]+ fn hyphenated_token_quotes_as_subtoken_phrase() {+ assert_eq!(+ prepare_fts5_query("ui-refresh"),+ "(\"ui-refresh\" OR \"ui refresh\")"+ );+ assert_eq!(+ prepare_fts5_query("scroll-restoration"),+ "(\"scroll-restoration\" OR \"scroll restoration\")"+ );}/// The FTS5 prefix operator (`term*`) must survive — a trailing `*` is not@@ -140,9 +198,16 @@ mod tests {}#[test]- fn quotes_are_escaped() {- let q = quote_fts5_token(r#"say "hello""#);- assert_eq!(q, r#""say ""hello""""#);+ fn quote_emits_whole_and_subtoken_phrase() {+ // Punctuated identifier → both forms OR'd.+ assert_eq!(+ quote_fts5_token("ai-memory"),+ r#"("ai-memory" OR "ai memory")"#+ );+ // A literal-quote fragment keeps the simple escaped form (no expansion).+ assert_eq!(quote_fts5_token(r#"say "hello""#), r#""say ""hello""""#);+ // No punctuation → single phrase.+ assert_eq!(quote_fts5_token("handoff"), r#""handoff""#);}#[test]diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex b41aede0..aae56205 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -2332,4 +2332,53 @@ mod tests {proves nothing");}++ /// Regression for the live-found hyphen bug: searching `ui-refresh`+ /// returned nothing in prod even though+ /// `follow-ups/ui-refresh-scroll-restoration.md` exists. The first fix+ /// quoted it as `"ui-refresh"`, which **does not error but also does not+ /// match** the indexed `ui refresh` — only `"ui refresh"` (sub-token+ /// phrase) does. The string-level test can't see this; this drives real+ /// FTS5 against the real `path_search` index. It would have caught the+ /// bug the dotted-only fix left behind.+ #[test]+ fn hyphenated_filename_search_matches_indexed_path() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ upsert_page(+ &mut conn,+ &page(+ ws,+ proj,+ "follow-ups/ui-refresh-scroll-restoration.md",+ "body text",+ ),+ )+ .unwrap();++ let hits = fts_match_paths(&conn, "ui-refresh")+ .expect("hyphenated search must not raise an FTS5 syntax error");+ assert!(+ hits.iter()+ .any(|p| p == "follow-ups/ui-refresh-scroll-restoration.md"),+ "search for `ui-refresh` should match the indexed path; got {hits:?}"+ );++ // Pin the exact FTS5 quirk the fix works around: the keeps-the-hyphen+ // phrase matches nothing, the spaces phrase matches. If this ever+ // flips, the sub-token quoting is no longer load-bearing.+ let count = |q: &str| -> i64 {+ conn.query_row(+ "SELECT count(*) FROM pages_fts WHERE pages_fts MATCH ?1",+ params![q],+ |r| r.get(0),+ )+ .unwrap()+ };+ assert_eq!(+ count("\"ui-refresh\""),+ 0,+ "kept-hyphen phrase must not match"+ );+ assert_eq!(count("\"ui refresh\""), 1, "sub-token phrase must match");+ }}