Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(store): keep title and body column filters on the CJK stream

Rudy Mizrahi Celekli568093500ec5
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rs
index b7b675c9..3cdd33d1 100644
--- a/crates/ai-memory-store/src/fts_query.rs
+++ b/crates/ai-memory-store/src/fts_query.rs
@@ -233,19 +233,32 @@ pub(crate) fn cjk_index_terms(raw: &str) -> String {
/// `None` is the important common case: callers then execute the existing SQL
/// byte-for-byte. A multi-scalar run becomes an exact phrase of overlapping
/// bigrams; a single scalar uses the unigram fallback.
+///
+/// A leading `title:` / `body:` qualifier is carried onto the run's phrase.
+/// This stream is auxiliary — callers UNION it into the primary stream — so a
+/// qualifier dropped here searches every column of `pages_cjk_fts`, including
+/// `path_search`, and the union widens a query the user deliberately narrowed.
+/// The two names are the ones [`prepare_fts5_token`] accepts; any other bare
+/// column is left alone, because the primary stream neutralises it too.
#[must_use]
pub(crate) fn prepare_cjk_fts5_query(raw: &str) -> Option<String> {
let bounded = bounded_query_text(raw);
let mut phrases = Vec::new();
- let mut run = Vec::new();
- for ch in bounded.chars() {
- if is_cjk(ch) {
- run.push(ch);
- } else {
- append_cjk_query(&mut phrases, &mut run);
+ for token in bounded.split_whitespace() {
+ let (column, text) = match token.split_once(':') {
+ Some((column @ ("title" | "body"), text)) => (Some(column), text),
+ _ => (None, token),
+ };
+ let mut run = Vec::new();
+ for ch in text.chars() {
+ if is_cjk(ch) {
+ run.push(ch);
+ } else {
+ append_cjk_query(&mut phrases, &mut run, column);
+ }
}
+ append_cjk_query(&mut phrases, &mut run, column);
}
- append_cjk_query(&mut phrases, &mut run);
(!phrases.is_empty()).then(|| phrases.join(" OR "))
}
@@ -258,19 +271,20 @@ fn append_cjk_terms(terms: &mut Vec<String>, run: &mut Vec<char>) {
run.clear();
}
-fn append_cjk_query(phrases: &mut Vec<String>, run: &mut Vec<char>) {
+fn append_cjk_query(phrases: &mut Vec<String>, run: &mut Vec<char>, column: Option<&str>) {
if run.is_empty() {
return;
}
+ let prefix = column.map_or_else(String::new, |column| format!("{column}:"));
if run.len() == 1 {
- phrases.push(format!("\"{}\"", run[0]));
+ phrases.push(format!("{prefix}\"{}\"", run[0]));
} else {
let bigrams = run
.windows(2)
.map(|pair| pair.iter().collect::<String>())
.collect::<Vec<_>>()
.join(" ");
- phrases.push(format!("\"{bigrams}\""));
+ phrases.push(format!("{prefix}\"{bigrams}\""));
}
run.clear();
}
@@ -504,6 +518,29 @@ mod tests {
assert_eq!(prepare_cjk_fts5_query("latin only"), None);
}
+ #[test]
+ fn cjk_queries_keep_a_known_column_filter() {
+ let title = prepare_cjk_fts5_query("title:支付服务").unwrap();
+ assert_eq!(title, "title:\"支付 付服 服务\"");
+ assert_parses(&title);
+ let body = prepare_cjk_fts5_query("body:支付服务").unwrap();
+ assert_eq!(body, "body:\"支付 付服 服务\"");
+ assert_parses(&body);
+ let single = prepare_cjk_fts5_query("title:支").unwrap();
+ assert_eq!(single, "title:\"支\"");
+ assert_parses(&single);
+ // Control: the bare query is untouched, and a column FTS5 does not
+ // know stays unqualified here exactly as the primary stream
+ // neutralises it, rather than being emitted as invalid syntax.
+ assert_eq!(
+ prepare_cjk_fts5_query("支付服务"),
+ Some("\"支付 付服 服务\"".to_string())
+ );
+ let unknown = prepare_cjk_fts5_query("author:支付服务").unwrap();
+ assert_eq!(unknown, "\"支付 付服 服务\"");
+ assert_parses(&unknown);
+ }
+
#[test]
fn cjk_query_input_is_bounded_before_expansion() {
let query = prepare_cjk_fts5_query(&"日".repeat(10_000)).unwrap();
diff --git a/crates/ai-memory-store/tests/suite/cjk_search.rs b/crates/ai-memory-store/tests/suite/cjk_search.rs
index 90d2db7e..eef56744 100644
--- a/crates/ai-memory-store/tests/suite/cjk_search.rs
+++ b/crates/ai-memory-store/tests/suite/cjk_search.rs
@@ -236,3 +236,66 @@ async fn superseded_cjk_text_is_hidden_and_startup_repairs_missing_index_rows()
assert_eq!(after_rebuild.len(), 1);
assert_eq!(after_rebuild[0].path.as_str(), "repair.md");
}
+
+#[tokio::test]
+async fn cjk_stream_honours_a_title_column_filter() {
+ let tmp = tempfile::tempdir().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (ws, project) = scope(&store, "default", "app").await;
+
+ store
+ .writer
+ .upsert_page(page(
+ ws,
+ project,
+ "titled.md",
+ "支付服务概要",
+ "English body",
+ ))
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(page(
+ ws,
+ project,
+ "bodied.md",
+ "Callback notes",
+ "启用微信支付服务后检查回调。",
+ ))
+ .await
+ .unwrap();
+
+ // Control: both pages carry 支付服务 in an unspaced run and both are
+ // reachable through the CJK stream, so the narrowed search below failing
+ // to return one of them means the column filter held.
+ let unqualified = store
+ .reader
+ .search_pages_for_project(ws, project, "支付服务".into(), 10, None)
+ .await
+ .unwrap();
+ let paths: Vec<_> = unqualified.iter().map(|hit| hit.path.as_str()).collect();
+ assert!(
+ paths.contains(&"titled.md"),
+ "control search failed: {paths:?}"
+ );
+ assert!(
+ paths.contains(&"bodied.md"),
+ "control search failed: {paths:?}"
+ );
+
+ let titled = store
+ .reader
+ .search_pages_for_project(ws, project, "title:支付服务".into(), 10, None)
+ .await
+ .unwrap();
+ let paths: Vec<_> = titled.iter().map(|hit| hit.path.as_str()).collect();
+ assert!(
+ paths.contains(&"titled.md"),
+ "the title match lost its page: {paths:?}"
+ );
+ assert!(
+ !paths.contains(&"bodied.md"),
+ "the CJK stream ignored the title filter: {paths:?}"
+ );
+}