Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(fts): match hyphenated terms against both content and path index

Djalma Júnior5b45b687d3de
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rs
index 9200d8ca..9c787705 100644
--- a/crates/ai-memory-store/src/fts_query.rs
+++ b/crates/ai-memory-store/src/fts_query.rs
@@ -76,9 +76,36 @@ fn should_quote_fts5_token(token: &str) -> bool {
}
fn quote_fts5_token(token: &str) -> String {
- // FTS5 escapes `"` inside a quoted token by doubling it.
- let escaped = token.replace('"', "\"\"");
- format!("\"{escaped}\"")
+ // FTS5 escapes `"` by doubling it. A token carrying a literal quote is an
+ // explicit-phrase fragment — keep the simple escaped form (don't expand it).
+ if token.contains('"') {
+ return format!("\"{}\"", token.replace('"', "\"\""));
+ }
+ // Otherwise emit BOTH the whole token and a punctuation-stripped sub-token
+ // phrase, OR'd, because the content tokenizer and the path index disagree
+ // on punctuation:
+ // tokenize = "unicode61 remove_diacritics 2 tokenchars '/_-'"
+ // keeps `/ _ -` INSIDE tokens (so a body mention of `ai-memory` indexes as
+ // the single token `ai-memory`), while `ops::path_search_text` pre-expands
+ // `/ . - _` to spaces in the path index (so a path `ui-refresh-…` indexes
+ // the sub-tokens `ui`, `refresh`, …). `.` is a separator either way.
+ // Neither form alone matches both: `"ai-memory"` matches the content token
+ // but not the split path index; `"ai memory"` matches the path but not the
+ // content token. OR-ing the two makes a search for `ai-memory` / `ui-refresh`
+ // hit whichever surface indexed it. (Quoting both also neutralises the
+ // punctuation that would otherwise be FTS5 query grammar — the original
+ // `current.md` → `syntax error` bug.) With no punctuation the two coincide
+ // and we emit a single phrase.
+ let split = token
+ .chars()
+ .map(|c| if c.is_ascii_punctuation() { ' ' } else { c })
+ .collect::<String>();
+ let split = split.split_whitespace().collect::<Vec<_>>().join(" ");
+ if split.is_empty() || split == token {
+ format!("\"{token}\"")
+ } else {
+ format!("(\"{token}\" OR \"{split}\")")
+ }
}
#[cfg(test)]
@@ -88,8 +115,11 @@ mod tests {
#[test]
fn colon_is_not_column_syntax() {
// Bare multi-word → OR-joined (no explicit operator present).
+ // `ai-memory` expands to BOTH the whole token (matches content) and
+ // the sub-token phrase (matches the split path index) — see
+ // `quote_fts5_token`.
let q = prepare_fts5_query("pick: handoff ai-memory");
- assert_eq!(q, "\"pick\" OR handoff OR \"ai-memory\"");
+ assert_eq!(q, "\"pick\" OR handoff OR (\"ai-memory\" OR \"ai memory\")");
}
#[test]
@@ -122,9 +152,37 @@ mod tests {
/// `current` + `md` are adjacent in the indexed path).
#[test]
fn dotted_filename_token_is_quoted() {
- assert_eq!(prepare_fts5_query("current.md"), "\"current.md\"");
- assert_eq!(prepare_fts5_query("00-index.md"), "\"00-index.md\"");
- assert_eq!(prepare_fts5_query("a/b/c.md"), "\"a/b/c.md\"");
+ // Whole token OR sub-token phrase. The split form (`current md`)
+ // matches the tokenised path; the whole form covers content tokens.
+ assert_eq!(
+ prepare_fts5_query("current.md"),
+ "(\"current.md\" OR \"current md\")"
+ );
+ assert_eq!(
+ prepare_fts5_query("00-index.md"),
+ "(\"00-index.md\" OR \"00 index md\")"
+ );
+ assert_eq!(
+ prepare_fts5_query("a/b/c.md"),
+ "(\"a/b/c.md\" OR \"a b c md\")"
+ );
+ }
+
+ /// Regression for the live-found bug: searching `ui-refresh` returned
+ /// nothing even though `follow-ups/ui-refresh-scroll-restoration.md`
+ /// exists. The old quoting produced `"ui-refresh"`, which FTS5 does NOT
+ /// match against the indexed `ui refresh`; the sub-token phrase
+ /// `"ui refresh"` does. See the real-FTS5 test in `ops.rs`.
+ #[test]
+ fn hyphenated_token_quotes_as_subtoken_phrase() {
+ assert_eq!(
+ prepare_fts5_query("ui-refresh"),
+ "(\"ui-refresh\" OR \"ui refresh\")"
+ );
+ assert_eq!(
+ prepare_fts5_query("scroll-restoration"),
+ "(\"scroll-restoration\" OR \"scroll restoration\")"
+ );
}
/// The FTS5 prefix operator (`term*`) must survive — a trailing `*` is not
@@ -140,9 +198,16 @@ mod tests {
}
#[test]
- fn quotes_are_escaped() {
- let q = quote_fts5_token(r#"say "hello""#);
- assert_eq!(q, r#""say ""hello""""#);
+ fn quote_emits_whole_and_subtoken_phrase() {
+ // Punctuated identifier → both forms OR'd.
+ assert_eq!(
+ quote_fts5_token("ai-memory"),
+ r#"("ai-memory" OR "ai memory")"#
+ );
+ // A literal-quote fragment keeps the simple escaped form (no expansion).
+ assert_eq!(quote_fts5_token(r#"say "hello""#), r#""say ""hello""""#);
+ // No punctuation → single phrase.
+ assert_eq!(quote_fts5_token("handoff"), r#""handoff""#);
}
#[test]
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index b41aede0..aae56205 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -2332,4 +2332,53 @@ mod tests {
proves nothing"
);
}
+
+ /// Regression for the live-found hyphen bug: searching `ui-refresh`
+ /// returned nothing in prod even though
+ /// `follow-ups/ui-refresh-scroll-restoration.md` exists. The first fix
+ /// quoted it as `"ui-refresh"`, which **does not error but also does not
+ /// match** the indexed `ui refresh` — only `"ui refresh"` (sub-token
+ /// phrase) does. The string-level test can't see this; this drives real
+ /// FTS5 against the real `path_search` index. It would have caught the
+ /// bug the dotted-only fix left behind.
+ #[test]
+ fn hyphenated_filename_search_matches_indexed_path() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ upsert_page(
+ &mut conn,
+ &page(
+ ws,
+ proj,
+ "follow-ups/ui-refresh-scroll-restoration.md",
+ "body text",
+ ),
+ )
+ .unwrap();
+
+ let hits = fts_match_paths(&conn, "ui-refresh")
+ .expect("hyphenated search must not raise an FTS5 syntax error");
+ assert!(
+ hits.iter()
+ .any(|p| p == "follow-ups/ui-refresh-scroll-restoration.md"),
+ "search for `ui-refresh` should match the indexed path; got {hits:?}"
+ );
+
+ // Pin the exact FTS5 quirk the fix works around: the keeps-the-hyphen
+ // phrase matches nothing, the spaces phrase matches. If this ever
+ // flips, the sub-token quoting is no longer load-bearing.
+ let count = |q: &str| -> i64 {
+ conn.query_row(
+ "SELECT count(*) FROM pages_fts WHERE pages_fts MATCH ?1",
+ params![q],
+ |r| r.get(0),
+ )
+ .unwrap()
+ };
+ assert_eq!(
+ count("\"ui-refresh\""),
+ 0,
+ "kept-hyphen phrase must not match"
+ );
+ assert_eq!(count("\"ui refresh\""), 1, "sub-token phrase must match");
+ }
}