Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(store): degrade invalid FTS5 operator mixes to a quoted bag of words

AkitaOnRails3d1128fffd83
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rs
index 9c787705..11844c4c 100644
--- a/crates/ai-memory-store/src/fts_query.rs
+++ b/crates/ai-memory-store/src/fts_query.rs
@@ -34,7 +34,54 @@ pub fn prepare_fts5_query(raw: &str) -> String {
return String::new();
}
let separator = if explicit_syntax { " " } else { " OR " };
- tokens.join(separator)
+ let candidate = tokens.join(separator);
+ // Natural language trips the explicit-syntax path constantly — `(MoMA)`,
+ // `'quoted phrases'`, a sentence that happens to contain OR — and a
+ // preserved-verbatim mix can be grammatically invalid FTS5 (a quoted
+ // OR-group adjacent to a bare term has no implicit AND, for one). The
+ // MATCH must never error on user text, so validate the candidate against
+ // a real FTS5 parser and degrade to the always-valid quoted bag of words
+ // when it does not parse. Deliberate, well-formed operator queries pass
+ // validation and are preserved.
+ if fts5_query_parses(&candidate) {
+ return candidate;
+ }
+ raw.split_whitespace()
+ .flat_map(|token| {
+ // Re-tokenize with every operator neutralised: strip quotes and
+ // parens, then quote what remains.
+ let cleaned: String = token
+ .chars()
+ .map(|c| if matches!(c, '"' | '(' | ')') { ' ' } else { c })
+ .collect();
+ cleaned
+ .split_whitespace()
+ .map(quote_fts5_token)
+ .collect::<Vec<_>>()
+ })
+ .collect::<Vec<_>>()
+ .join(" OR ")
+}
+
+/// Whether FTS5's own parser accepts `query`. Uses a throwaway in-memory
+/// table (microseconds; search-path frequency, not ingest-path) so the
+/// judgement is the engine's, not a re-implementation of its grammar.
+fn fts5_query_parses(query: &str) -> bool {
+ let Ok(conn) = rusqlite::Connection::open_in_memory() else {
+ return false;
+ };
+ if conn
+ .execute_batch("CREATE VIRTUAL TABLE fts_probe USING fts5(title, body)")
+ .is_err()
+ {
+ return false;
+ }
+ conn.query_row(
+ "SELECT count(*) FROM fts_probe WHERE fts_probe MATCH ?1",
+ [query],
+ |row| row.get::<_, i64>(0),
+ )
+ .is_ok()
}
fn prepare_fts5_token(token: &str) -> Vec<String> {
@@ -112,6 +159,70 @@ fn quote_fts5_token(token: &str) -> String {
mod tests {
use super::*;
+ /// The engine, not this module, is the judge of validity: every
+ /// prepared query must MATCH without error on a real FTS5 table.
+ fn assert_parses(prepared: &str) {
+ let conn = rusqlite::Connection::open_in_memory().unwrap();
+ conn.execute_batch("CREATE VIRTUAL TABLE t USING fts5(title, body)")
+ .unwrap();
+ let res = conn.query_row(
+ "SELECT count(*) FROM t WHERE t MATCH ?1",
+ [prepared],
+ |row| row.get::<_, i64>(0),
+ );
+ assert!(res.is_ok(), "FTS5 rejected {prepared:?}: {res:?}");
+ }
+
+ /// Regression (found by the LongMemEval harness): a natural-language
+ /// question containing parens and apostrophe-quotes tripped the
+ /// explicit-syntax path and produced invalid FTS5 — `fts5: syntax
+ /// error near "OR"` — failing the whole memory_query call.
+ #[test]
+ fn natural_language_with_parens_and_quotes_never_errors() {
+ let raw = "How many days passed between my visit to the Museum of \
+ Modern Art (MoMA) and the 'Ancient Civilizations' exhibit \
+ at the Metropolitan Museum of Art?";
+ let prepared = prepare_fts5_query(raw);
+ assert_parses(&prepared);
+ // and the degraded form still carries the searchable words
+ assert!(prepared.contains("MoMA"));
+ assert!(prepared.contains("Civilizations"));
+ }
+
+ #[test]
+ fn hostile_operator_soup_never_errors() {
+ for raw in [
+ "((",
+ "\"unclosed phrase",
+ "NEAR(",
+ "OR",
+ "a OR",
+ "OR b",
+ "NOT",
+ ") misplaced (",
+ "col:val:col ( OR \" NEAR",
+ "\"\"\"",
+ ] {
+ let prepared = prepare_fts5_query(raw);
+ if !prepared.is_empty() {
+ assert_parses(&prepared);
+ }
+ }
+ }
+
+ #[test]
+ fn deliberate_well_formed_syntax_is_preserved() {
+ let prepared = prepare_fts5_query("title:handoff OR body:deploy");
+ assert_parses(&prepared);
+ assert_eq!(prepared, "title:handoff OR body:deploy");
+ // Quoted phrases were re-tokenized into escaped-quote form long
+ // before the validation fallback existed; the contract here is
+ // "still valid FTS5 with AND preserved", not byte identity.
+ let phrase = prepare_fts5_query("\"exact phrase\" AND deploy");
+ assert_parses(&phrase);
+ assert!(phrase.contains(" AND deploy"), "{phrase}");
+ }
+
#[test]
fn colon_is_not_column_syntax() {
// Bare multi-word → OR-joined (no explicit operator present).