Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(search): drop stopwords from bare-query OR-joins

AkitaOnRails0ac0dcf8f89a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76c4a93e..9bbbd10c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,6 +64,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`docs/okf.md` and `docs/MIGRATION-2.0.md`.
### Fixed
+- Natural-language searches no longer surface stopword trash: bare
+ queries drop English stopwords before the FTS5 OR-join, so a page
+ containing five "the"s cannot outrank the page whose content matches,
+ and pages matching only stopwords no longer appear at all. Quoted
+ phrases and explicit-operator queries are untouched, and an
+ all-stopword query still searches its literal terms. Guarded by a
+ CI-runnable search-quality suite asserting ranking usefulness, not
+ just machinery.
- `memory_query` no longer fails with `fts5: syntax error` when the query is
natural language containing parentheses, apostrophe-quoted phrases, or a
stray `OR`/`AND` (e.g. *"my visit to the Museum of Modern Art (MoMA) and
diff --git a/crates/ai-memory-consolidate/tests/search_quality.rs b/crates/ai-memory-consolidate/tests/search_quality.rs
new file mode 100644
index 00000000..9c43bff7
--- /dev/null
+++ b/crates/ai-memory-consolidate/tests/search_quality.rs
@@ -0,0 +1,169 @@
+//! Lexical-search usefulness suite.
+//!
+//! Same philosophy as `local_embeddings.rs` but CI-runnable (no model):
+//! these tests assert that search RESULTS serve the user, not merely
+//! that the machinery runs. The scenario is the one that exposed the
+//! problem live: with no stopword handling, a natural-language question
+//! OR-joined into FTS5 let a page containing five "the"s outrank the
+//! page whose content matched — and every page containing any stopword
+//! entered the result list as trash.
+
+use ai_memory_core::{PagePath, Tier};
+use ai_memory_store::Store;
+use ai_memory_wiki::{Wiki, WritePageRequest};
+use tempfile::TempDir;
+
+async fn seeded() -> (
+ TempDir,
+ Store,
+ ai_memory_core::WorkspaceId,
+ ai_memory_core::ProjectId,
+) {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ // Stopword-heavy bodies on purpose — the pre-fix failure needed
+ // nothing more than "the" repetition to produce trash rankings.
+ for (path, body) in [
+ (
+ "notes/changelog-style.md",
+ "Every new version of the app gets a changelog entry describing \
+ what the new version changed for app users.",
+ ),
+ (
+ "procedures/release.md",
+ "The release procedure: tag main, wait for the pipeline, then \
+ push the image to the registry and restart the compose stack.",
+ ),
+ (
+ "notes/pasta.md",
+ "Boil the water with plenty of the salt before adding the \
+ spaghetti to the pot.",
+ ),
+ (
+ "notes/howto-index.md",
+ "How do we keep this wiki organised: one page per concept.",
+ ),
+ ] {
+ wiki.write_page(WritePageRequest {
+ workspace_id: ws,
+ project_id: proj,
+ path: PagePath::new(path).unwrap(),
+ frontmatter: serde_json::json!({"title": path}),
+ body: body.into(),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: None,
+ admission_ctx: None,
+ author_id: None,
+ actor: ai_memory_core::ActorContext::anonymous(),
+ })
+ .await
+ .unwrap();
+ }
+ (tmp, store, ws, proj)
+}
+
+async fn fts(
+ store: &Store,
+ ws: ai_memory_core::WorkspaceId,
+ proj: ai_memory_core::ProjectId,
+ q: &str,
+) -> Vec<String> {
+ store
+ .reader
+ .hybrid_search(
+ ws,
+ proj,
+ q.to_string(),
+ None,
+ String::new(),
+ String::new(),
+ 0,
+ 10,
+ None,
+ )
+ .await
+ .unwrap()
+ .into_iter()
+ .map(|h| h.path.as_str().to_string())
+ .collect()
+}
+
+/// The anti-trash property: for a natural-language question, pages
+/// matching only the question's stopwords do not appear at all, and the
+/// page matching the CONTENT words ranks first.
+#[tokio::test]
+async fn natural_questions_rank_content_not_stopword_frequency() {
+ let (_tmp, store, ws, proj) = seeded().await;
+ let hits = fts(
+ &store,
+ ws,
+ proj,
+ "how do we deploy a new version of the app?",
+ )
+ .await;
+ assert_eq!(
+ hits.first().map(String::as_str),
+ Some("notes/changelog-style.md"),
+ "content-word matches must rank first: {hits:?}"
+ );
+ for trash in ["notes/pasta.md", "procedures/release.md"] {
+ assert!(
+ !hits.iter().any(|h| h == trash),
+ "{trash} matches only stopwords and must not appear: {hits:?}"
+ );
+ }
+}
+
+/// A query that is ONLY stopwords still searches the user's literal
+/// terms — an empty result page for "how do we" would be worse.
+#[tokio::test]
+async fn all_stopword_queries_fall_back_to_literal_terms() {
+ let (_tmp, store, ws, proj) = seeded().await;
+ let hits = fts(&store, ws, proj, "how do we").await;
+ assert!(
+ hits.iter().any(|h| h == "notes/howto-index.md"),
+ "literal stopword terms must still match when nothing else is \
+ in the query: {hits:?}"
+ );
+}
+
+/// Quoted phrases keep their stopwords — "the compose stack" is an
+/// exact-phrase request, not a bag of words.
+#[tokio::test]
+async fn quoted_phrases_keep_stopwords() {
+ let (_tmp, store, ws, proj) = seeded().await;
+ let hits = fts(&store, ws, proj, "\"restart the compose stack\"").await;
+ assert_eq!(
+ hits,
+ vec!["procedures/release.md".to_string()],
+ "exact phrase must match exactly one page"
+ );
+}
+
+/// Keyword queries (the common agent-issued shape) are unaffected:
+/// every content word still ORs.
+#[tokio::test]
+async fn keyword_queries_are_unchanged() {
+ let (_tmp, store, ws, proj) = seeded().await;
+ let hits = fts(&store, ws, proj, "changelog registry").await;
+ assert!(
+ hits.iter().any(|h| h == "notes/changelog-style.md"),
+ "{hits:?}"
+ );
+ assert!(
+ hits.iter().any(|h| h == "procedures/release.md"),
+ "{hits:?}"
+ );
+}
diff --git a/crates/ai-memory-store/src/fts_query.rs b/crates/ai-memory-store/src/fts_query.rs
index 11844c4c..1b5f9c06 100644
--- a/crates/ai-memory-store/src/fts_query.rs
+++ b/crates/ai-memory-store/src/fts_query.rs
@@ -28,8 +28,24 @@ pub fn prepare_fts5_query(raw: &str) -> String {
.any(|t| matches!(t, "OR" | "AND" | "NOT" | "NEAR"));
let tokens: Vec<String> = raw
.split_whitespace()
+ // Bare natural-language queries drop stopwords before the
+ // OR-join: with no tokenizer-level stopword list, "the/of/a"
+ // match nearly every page and BM25 term frequency lets a page
+ // with five "the"s outrank the page whose CONTENT matches (seen
+ // live: a release-procedure page beaten for a deploy question).
+ // Explicit-syntax queries and quoted phrases are untouched, and
+ // a query that is ONLY stopwords keeps them all — returning the
+ // user's literal terms beats returning nothing.
+ .filter(|t| explicit_syntax || !is_stopword(t))
.flat_map(prepare_fts5_token)
.collect();
+ let tokens = if tokens.is_empty() && !explicit_syntax {
+ raw.split_whitespace()
+ .flat_map(prepare_fts5_token)
+ .collect()
+ } else {
+ tokens
+ };
if tokens.is_empty() {
return String::new();
}
@@ -84,6 +100,73 @@ fn fts5_query_parses(query: &str) -> bool {
.is_ok()
}
+/// English stopwords excluded from bare-query OR-joins. Deliberately
+/// small and boring: high-document-frequency function words that carry
+/// no retrieval signal but dominate BM25 through term frequency. Words
+/// inside quoted phrases and explicit-operator queries never pass
+/// through this filter.
+fn is_stopword(token: &str) -> bool {
+ matches!(
+ token.to_ascii_lowercase().as_str(),
+ "a" | "an"
+ | "and"
+ | "are"
+ | "as"
+ | "at"
+ | "be"
+ | "been"
+ | "but"
+ | "by"
+ | "can"
+ | "could"
+ | "did"
+ | "do"
+ | "does"
+ | "for"
+ | "from"
+ | "had"
+ | "has"
+ | "have"
+ | "how"
+ | "i"
+ | "if"
+ | "in"
+ | "is"
+ | "it"
+ | "its"
+ | "me"
+ | "my"
+ | "of"
+ | "on"
+ | "or"
+ | "our"
+ | "she"
+ | "should"
+ | "so"
+ | "that"
+ | "the"
+ | "their"
+ | "them"
+ | "they"
+ | "this"
+ | "to"
+ | "was"
+ | "we"
+ | "were"
+ | "what"
+ | "when"
+ | "where"
+ | "which"
+ | "who"
+ | "why"
+ | "will"
+ | "with"
+ | "would"
+ | "you"
+ | "your"
+ )
+}
+
fn prepare_fts5_token(token: &str) -> Vec<String> {
if has_unknown_bare_column(token) {
return token