Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(store): skip the CJK stream for a query the expander would invert

Rudy Mizrahi Celeklicc0697b73fd9
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rs
index b7b675c9..872052fe 100644
--- a/crates/ai-memory-store/src/fts_query.rs
+++ b/crates/ai-memory-store/src/fts_query.rs
@@ -233,9 +233,24 @@ pub(crate) fn cjk_index_terms(raw: &str) -> String {
/// `None` is the important common case: callers then execute the existing SQL
/// byte-for-byte. A multi-scalar run becomes an exact phrase of overlapping
/// bigrams; a single scalar uses the unigram fallback.
+///
+/// `None` also covers a query carrying `AND` / `NOT` / `NEAR`. This stream is
+/// auxiliary: callers UNION it into the primary stream, so a run-based
+/// OR-join of a `東京 NOT 大阪` query fuses the excluded term's own pages back
+/// into the result set — the opposite of what the operator asked for, while
+/// the primary stream honours it verbatim. Expanding each operand and
+/// rebuilding the expression is the real fix; declining the stream meanwhile
+/// restores the pre-#1433 answer instead of an inverted one. `OR` stays,
+/// because the expander reproduces it faithfully.
#[must_use]
pub(crate) fn prepare_cjk_fts5_query(raw: &str) -> Option<String> {
let bounded = bounded_query_text(raw);
+ if bounded
+ .split_whitespace()
+ .any(|t| matches!(t, "AND" | "NOT" | "NEAR"))
+ {
+ return None;
+ }
let mut phrases = Vec::new();
let mut run = Vec::new();
for ch in bounded.chars() {
@@ -504,6 +519,20 @@ mod tests {
assert_eq!(prepare_cjk_fts5_query("latin only"), None);
}
+ #[test]
+ fn cjk_stream_declines_a_query_the_expander_would_invert() {
+ assert_eq!(prepare_cjk_fts5_query("東京 NOT 大阪"), None);
+ assert_eq!(prepare_cjk_fts5_query("東京 AND 大阪"), None);
+ assert_eq!(prepare_cjk_fts5_query("東京 NEAR 大阪"), None);
+ // Control: `OR` is the one operator the run expander reproduces, so
+ // the auxiliary stream still runs for it — and a lowercase `not` is
+ // not FTS5 syntax, so it must not suppress the stream either.
+ let both = Some("\"東京\" OR \"大阪\"".to_string());
+ assert_eq!(prepare_cjk_fts5_query("東京 OR 大阪"), both);
+ assert_eq!(prepare_cjk_fts5_query("東京 not 大阪"), both);
+ assert_parses(&prepare_cjk_fts5_query("東京 OR 大阪").unwrap());
+ }
+
#[test]
fn cjk_query_input_is_bounded_before_expansion() {
let query = prepare_cjk_fts5_query(&"日".repeat(10_000)).unwrap();
diff --git a/crates/ai-memory-store/tests/suite/cjk_search.rs b/crates/ai-memory-store/tests/suite/cjk_search.rs
index 90d2db7e..d2a0287e 100644
--- a/crates/ai-memory-store/tests/suite/cjk_search.rs
+++ b/crates/ai-memory-store/tests/suite/cjk_search.rs
@@ -236,3 +236,58 @@ async fn superseded_cjk_text_is_hidden_and_startup_repairs_missing_index_rows()
assert_eq!(after_rebuild.len(), 1);
assert_eq!(after_rebuild[0].path.as_str(), "repair.md");
}
+
+#[tokio::test]
+async fn cjk_stream_does_not_fuse_back_a_not_excluded_term() {
+ let tmp = tempfile::tempdir().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (ws, project) = scope(&store, "default", "app").await;
+
+ store
+ .writer
+ .upsert_page(page(ws, project, "tokyo.md", "東京", "office notes"))
+ .await
+ .unwrap();
+ // 大阪 only ever appears inside an unspaced run, so unicode61 gives the
+ // primary stream no token to match: this page is reachable through the
+ // auxiliary CJK stream alone.
+ store
+ .writer
+ .upsert_page(page(
+ ws,
+ project,
+ "osaka.md",
+ "Kansai",
+ "関西の大阪支社について",
+ ))
+ .await
+ .unwrap();
+
+ // Control: the excluded page IS indexed and IS findable on its own, so an
+ // absence below means the operator was honoured, not that nothing matched.
+ let osaka = store
+ .reader
+ .search_pages_for_project(ws, project, "大阪".into(), 10, None)
+ .await
+ .unwrap();
+ let paths: Vec<_> = osaka.iter().map(|hit| hit.path.as_str()).collect();
+ assert!(
+ paths.contains(&"osaka.md"),
+ "control search failed: {paths:?}"
+ );
+
+ let excluded = store
+ .reader
+ .search_pages_for_project(ws, project, "東京 NOT 大阪".into(), 10, None)
+ .await
+ .unwrap();
+ let paths: Vec<_> = excluded.iter().map(|hit| hit.path.as_str()).collect();
+ assert!(
+ paths.contains(&"tokyo.md"),
+ "the requested term lost its page: {paths:?}"
+ );
+ assert!(
+ !paths.contains(&"osaka.md"),
+ "the CJK stream fused back the NOT-excluded term: {paths:?}"
+ );
+}