Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(store): degrade invalid FTS5 operator mixes to a quoted bag of words
AkitaOnRails3d1128fffd83
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rsindex 9c787705..11844c4c 100644--- a/crates/ai-memory-store/src/fts_query.rs+++ b/crates/ai-memory-store/src/fts_query.rs@@ -34,7 +34,54 @@ pub fn prepare_fts5_query(raw: &str) -> String {return String::new();}let separator = if explicit_syntax { " " } else { " OR " };- tokens.join(separator)+ let candidate = tokens.join(separator);+ // Natural language trips the explicit-syntax path constantly — `(MoMA)`,+ // `'quoted phrases'`, a sentence that happens to contain OR — and a+ // preserved-verbatim mix can be grammatically invalid FTS5 (a quoted+ // OR-group adjacent to a bare term has no implicit AND, for one). The+ // MATCH must never error on user text, so validate the candidate against+ // a real FTS5 parser and degrade to the always-valid quoted bag of words+ // when it does not parse. Deliberate, well-formed operator queries pass+ // validation and are preserved.+ if fts5_query_parses(&candidate) {+ return candidate;+ }+ raw.split_whitespace()+ .flat_map(|token| {+ // Re-tokenize with every operator neutralised: strip quotes and+ // parens, then quote what remains.+ let cleaned: String = token+ .chars()+ .map(|c| if matches!(c, '"' | '(' | ')') { ' ' } else { c })+ .collect();+ cleaned+ .split_whitespace()+ .map(quote_fts5_token)+ .collect::<Vec<_>>()+ })+ .collect::<Vec<_>>()+ .join(" OR ")+}++/// Whether FTS5's own parser accepts `query`. Uses a throwaway in-memory+/// table (microseconds; search-path frequency, not ingest-path) so the+/// judgement is the engine's, not a re-implementation of its grammar.+fn fts5_query_parses(query: &str) -> bool {+ let Ok(conn) = rusqlite::Connection::open_in_memory() else {+ return false;+ };+ if conn+ .execute_batch("CREATE VIRTUAL TABLE fts_probe USING fts5(title, body)")+ .is_err()+ {+ return false;+ }+ conn.query_row(+ "SELECT count(*) FROM fts_probe WHERE fts_probe MATCH ?1",+ [query],+ |row| row.get::<_, i64>(0),+ )+ .is_ok()}fn prepare_fts5_token(token: &str) -> Vec<String> {@@ -112,6 +159,70 @@ fn quote_fts5_token(token: &str) -> String {mod tests {use super::*;+ /// The engine, not this module, is the judge of validity: every+ /// prepared query must MATCH without error on a real FTS5 table.+ fn assert_parses(prepared: &str) {+ let conn = rusqlite::Connection::open_in_memory().unwrap();+ conn.execute_batch("CREATE VIRTUAL TABLE t USING fts5(title, body)")+ .unwrap();+ let res = conn.query_row(+ "SELECT count(*) FROM t WHERE t MATCH ?1",+ [prepared],+ |row| row.get::<_, i64>(0),+ );+ assert!(res.is_ok(), "FTS5 rejected {prepared:?}: {res:?}");+ }++ /// Regression (found by the LongMemEval harness): a natural-language+ /// question containing parens and apostrophe-quotes tripped the+ /// explicit-syntax path and produced invalid FTS5 — `fts5: syntax+ /// error near "OR"` — failing the whole memory_query call.+ #[test]+ fn natural_language_with_parens_and_quotes_never_errors() {+ let raw = "How many days passed between my visit to the Museum of \+ Modern Art (MoMA) and the 'Ancient Civilizations' exhibit \+ at the Metropolitan Museum of Art?";+ let prepared = prepare_fts5_query(raw);+ assert_parses(&prepared);+ // and the degraded form still carries the searchable words+ assert!(prepared.contains("MoMA"));+ assert!(prepared.contains("Civilizations"));+ }++ #[test]+ fn hostile_operator_soup_never_errors() {+ for raw in [+ "((",+ "\"unclosed phrase",+ "NEAR(",+ "OR",+ "a OR",+ "OR b",+ "NOT",+ ") misplaced (",+ "col:val:col ( OR \" NEAR",+ "\"\"\"",+ ] {+ let prepared = prepare_fts5_query(raw);+ if !prepared.is_empty() {+ assert_parses(&prepared);+ }+ }+ }++ #[test]+ fn deliberate_well_formed_syntax_is_preserved() {+ let prepared = prepare_fts5_query("title:handoff OR body:deploy");+ assert_parses(&prepared);+ assert_eq!(prepared, "title:handoff OR body:deploy");+ // Quoted phrases were re-tokenized into escaped-quote form long+ // before the validation fallback existed; the contract here is+ // "still valid FTS5 with AND preserved", not byte identity.+ let phrase = prepare_fts5_query("\"exact phrase\" AND deploy");+ assert_parses(&phrase);+ assert!(phrase.contains(" AND deploy"), "{phrase}");+ }+#[test]fn colon_is_not_column_syntax() {// Bare multi-word → OR-joined (no explicit operator present).