Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(store): keep title and body column filters on the CJK stream
Rudy Mizrahi Celekli568093500ec5
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rsindex b7b675c9..3cdd33d1 100644--- a/crates/ai-memory-store/src/fts_query.rs+++ b/crates/ai-memory-store/src/fts_query.rs@@ -233,19 +233,32 @@ pub(crate) fn cjk_index_terms(raw: &str) -> String {/// `None` is the important common case: callers then execute the existing SQL/// byte-for-byte. A multi-scalar run becomes an exact phrase of overlapping/// bigrams; a single scalar uses the unigram fallback.+///+/// A leading `title:` / `body:` qualifier is carried onto the run's phrase.+/// This stream is auxiliary — callers UNION it into the primary stream — so a+/// qualifier dropped here searches every column of `pages_cjk_fts`, including+/// `path_search`, and the union widens a query the user deliberately narrowed.+/// The two names are the ones [`prepare_fts5_token`] accepts; any other bare+/// column is left alone, because the primary stream neutralises it too.#[must_use]pub(crate) fn prepare_cjk_fts5_query(raw: &str) -> Option<String> {let bounded = bounded_query_text(raw);let mut phrases = Vec::new();- let mut run = Vec::new();- for ch in bounded.chars() {- if is_cjk(ch) {- run.push(ch);- } else {- append_cjk_query(&mut phrases, &mut run);+ for token in bounded.split_whitespace() {+ let (column, text) = match token.split_once(':') {+ Some((column @ ("title" | "body"), text)) => (Some(column), text),+ _ => (None, token),+ };+ let mut run = Vec::new();+ for ch in text.chars() {+ if is_cjk(ch) {+ run.push(ch);+ } else {+ append_cjk_query(&mut phrases, &mut run, column);+ }}+ append_cjk_query(&mut phrases, &mut run, column);}- append_cjk_query(&mut phrases, &mut run);(!phrases.is_empty()).then(|| phrases.join(" OR "))}@@ -258,19 +271,20 @@ fn append_cjk_terms(terms: &mut Vec<String>, run: &mut Vec<char>) {run.clear();}-fn append_cjk_query(phrases: &mut Vec<String>, run: &mut Vec<char>) {+fn append_cjk_query(phrases: &mut Vec<String>, run: &mut Vec<char>, column: Option<&str>) {if run.is_empty() {return;}+ let prefix = column.map_or_else(String::new, |column| format!("{column}:"));if run.len() == 1 {- phrases.push(format!("\"{}\"", run[0]));+ phrases.push(format!("{prefix}\"{}\"", run[0]));} else {let bigrams = run.windows(2).map(|pair| pair.iter().collect::<String>()).collect::<Vec<_>>().join(" ");- phrases.push(format!("\"{bigrams}\""));+ phrases.push(format!("{prefix}\"{bigrams}\""));}run.clear();}@@ -504,6 +518,29 @@ mod tests {assert_eq!(prepare_cjk_fts5_query("latin only"), None);}+ #[test]+ fn cjk_queries_keep_a_known_column_filter() {+ let title = prepare_cjk_fts5_query("title:支付服务").unwrap();+ assert_eq!(title, "title:\"支付 付服 服务\"");+ assert_parses(&title);+ let body = prepare_cjk_fts5_query("body:支付服务").unwrap();+ assert_eq!(body, "body:\"支付 付服 服务\"");+ assert_parses(&body);+ let single = prepare_cjk_fts5_query("title:支").unwrap();+ assert_eq!(single, "title:\"支\"");+ assert_parses(&single);+ // Control: the bare query is untouched, and a column FTS5 does not+ // know stays unqualified here exactly as the primary stream+ // neutralises it, rather than being emitted as invalid syntax.+ assert_eq!(+ prepare_cjk_fts5_query("支付服务"),+ Some("\"支付 付服 服务\"".to_string())+ );+ let unknown = prepare_cjk_fts5_query("author:支付服务").unwrap();+ assert_eq!(unknown, "\"支付 付服 服务\"");+ assert_parses(&unknown);+ }+#[test]fn cjk_query_input_is_bounded_before_expansion() {let query = prepare_cjk_fts5_query(&"日".repeat(10_000)).unwrap();diff --git a/crates/ai-memory-store/tests/suite/cjk_search.rs b/crates/ai-memory-store/tests/suite/cjk_search.rsindex 90d2db7e..eef56744 100644--- a/crates/ai-memory-store/tests/suite/cjk_search.rs+++ b/crates/ai-memory-store/tests/suite/cjk_search.rs@@ -236,3 +236,66 @@ async fn superseded_cjk_text_is_hidden_and_startup_repairs_missing_index_rows()assert_eq!(after_rebuild.len(), 1);assert_eq!(after_rebuild[0].path.as_str(), "repair.md");}++#[tokio::test]+async fn cjk_stream_honours_a_title_column_filter() {+ let tmp = tempfile::tempdir().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let (ws, project) = scope(&store, "default", "app").await;++ store+ .writer+ .upsert_page(page(+ ws,+ project,+ "titled.md",+ "支付服务概要",+ "English body",+ ))+ .await+ .unwrap();+ store+ .writer+ .upsert_page(page(+ ws,+ project,+ "bodied.md",+ "Callback notes",+ "启用微信支付服务后检查回调。",+ ))+ .await+ .unwrap();++ // Control: both pages carry 支付服务 in an unspaced run and both are+ // reachable through the CJK stream, so the narrowed search below failing+ // to return one of them means the column filter held.+ let unqualified = store+ .reader+ .search_pages_for_project(ws, project, "支付服务".into(), 10, None)+ .await+ .unwrap();+ let paths: Vec<_> = unqualified.iter().map(|hit| hit.path.as_str()).collect();+ assert!(+ paths.contains(&"titled.md"),+ "control search failed: {paths:?}"+ );+ assert!(+ paths.contains(&"bodied.md"),+ "control search failed: {paths:?}"+ );++ let titled = store+ .reader+ .search_pages_for_project(ws, project, "title:支付服务".into(), 10, None)+ .await+ .unwrap();+ let paths: Vec<_> = titled.iter().map(|hit| hit.path.as_str()).collect();+ assert!(+ paths.contains(&"titled.md"),+ "the title match lost its page: {paths:?}"+ );+ assert!(+ !paths.contains(&"bodied.md"),+ "the CJK stream ignored the title filter: {paths:?}"+ );+}