Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(store): skip the CJK stream for a query the expander would invert
Rudy Mizrahi Celeklicc0697b73fd9
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rsindex b7b675c9..872052fe 100644--- a/crates/ai-memory-store/src/fts_query.rs+++ b/crates/ai-memory-store/src/fts_query.rs@@ -233,9 +233,24 @@ pub(crate) fn cjk_index_terms(raw: &str) -> String {/// `None` is the important common case: callers then execute the existing SQL/// byte-for-byte. A multi-scalar run becomes an exact phrase of overlapping/// bigrams; a single scalar uses the unigram fallback.+///+/// `None` also covers a query carrying `AND` / `NOT` / `NEAR`. This stream is+/// auxiliary: callers UNION it into the primary stream, so a run-based+/// OR-join of a `東京 NOT 大阪` query fuses the excluded term's own pages back+/// into the result set — the opposite of what the operator asked for, while+/// the primary stream honours it verbatim. Expanding each operand and+/// rebuilding the expression is the real fix; declining the stream meanwhile+/// restores the pre-#1433 answer instead of an inverted one. `OR` stays,+/// because the expander reproduces it faithfully.#[must_use]pub(crate) fn prepare_cjk_fts5_query(raw: &str) -> Option<String> {let bounded = bounded_query_text(raw);+ if bounded+ .split_whitespace()+ .any(|t| matches!(t, "AND" | "NOT" | "NEAR"))+ {+ return None;+ }let mut phrases = Vec::new();let mut run = Vec::new();for ch in bounded.chars() {@@ -504,6 +519,20 @@ mod tests {assert_eq!(prepare_cjk_fts5_query("latin only"), None);}+ #[test]+ fn cjk_stream_declines_a_query_the_expander_would_invert() {+ assert_eq!(prepare_cjk_fts5_query("東京 NOT 大阪"), None);+ assert_eq!(prepare_cjk_fts5_query("東京 AND 大阪"), None);+ assert_eq!(prepare_cjk_fts5_query("東京 NEAR 大阪"), None);+ // Control: `OR` is the one operator the run expander reproduces, so+ // the auxiliary stream still runs for it — and a lowercase `not` is+ // not FTS5 syntax, so it must not suppress the stream either.+ let both = Some("\"東京\" OR \"大阪\"".to_string());+ assert_eq!(prepare_cjk_fts5_query("東京 OR 大阪"), both);+ assert_eq!(prepare_cjk_fts5_query("東京 not 大阪"), both);+ assert_parses(&prepare_cjk_fts5_query("東京 OR 大阪").unwrap());+ }+#[test]fn cjk_query_input_is_bounded_before_expansion() {let query = prepare_cjk_fts5_query(&"日".repeat(10_000)).unwrap();diff --git a/crates/ai-memory-store/tests/suite/cjk_search.rs b/crates/ai-memory-store/tests/suite/cjk_search.rsindex 90d2db7e..d2a0287e 100644--- a/crates/ai-memory-store/tests/suite/cjk_search.rs+++ b/crates/ai-memory-store/tests/suite/cjk_search.rs@@ -236,3 +236,58 @@ async fn superseded_cjk_text_is_hidden_and_startup_repairs_missing_index_rows()assert_eq!(after_rebuild.len(), 1);assert_eq!(after_rebuild[0].path.as_str(), "repair.md");}++#[tokio::test]+async fn cjk_stream_does_not_fuse_back_a_not_excluded_term() {+ let tmp = tempfile::tempdir().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let (ws, project) = scope(&store, "default", "app").await;++ store+ .writer+ .upsert_page(page(ws, project, "tokyo.md", "東京", "office notes"))+ .await+ .unwrap();+ // 大阪 only ever appears inside an unspaced run, so unicode61 gives the+ // primary stream no token to match: this page is reachable through the+ // auxiliary CJK stream alone.+ store+ .writer+ .upsert_page(page(+ ws,+ project,+ "osaka.md",+ "Kansai",+ "関西の大阪支社について",+ ))+ .await+ .unwrap();++ // Control: the excluded page IS indexed and IS findable on its own, so an+ // absence below means the operator was honoured, not that nothing matched.+ let osaka = store+ .reader+ .search_pages_for_project(ws, project, "大阪".into(), 10, None)+ .await+ .unwrap();+ let paths: Vec<_> = osaka.iter().map(|hit| hit.path.as_str()).collect();+ assert!(+ paths.contains(&"osaka.md"),+ "control search failed: {paths:?}"+ );++ let excluded = store+ .reader+ .search_pages_for_project(ws, project, "東京 NOT 大阪".into(), 10, None)+ .await+ .unwrap();+ let paths: Vec<_> = excluded.iter().map(|hit| hit.path.as_str()).collect();+ assert!(+ paths.contains(&"tokyo.md"),+ "the requested term lost its page: {paths:?}"+ );+ assert!(+ !paths.contains(&"osaka.md"),+ "the CJK stream fused back the NOT-excluded term: {paths:?}"+ );+}