Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): #797 must_not-only bool no longer projects to a pure-negative FTS bool (#798)

xerj-org0325552b1702
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index edde8ecbb..a7ce4e0b6 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1659,6 +1659,65 @@ mod flush_publication_recovery_tests {
}
}
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn mustnot_only_prefix_keeps_nonmatching_docs_after_flush() {
+ // #797: a bool with ONLY a must_not (no must/should/filter) containing a
+ // prefix/wildcard leaf returns zero hits after flush — the segment
+ // planner lets the negative leaf drive candidate generation, so the
+ // non-matching docs never enter the set. `term` is unaffected. Base set
+ // must be all docs, must_not subtracting only its matches.
+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("k", FieldType::Keyword))
+ .unwrap();
+ engine.create_index("mn797", schema).unwrap();
+ let idx = engine.get_index("mn797").unwrap();
+ idx.abort_background_tasks();
+ // Data is case-fold-invariant for prefix "ap"/"app*": only "apple"
+ // starts with it under ANY casing, so the correct answer does not
+ // depend on the separate #794 keyword-prefix case-sensitivity fix.
+ for (id, k) in [("1", "apple"), ("2", "grape"), ("3", "banana")] {
+ idx.index_document(Some(id.into()), json!({ "k": k }))
+ .await
+ .unwrap();
+ }
+ let run = |idx: std::sync::Arc<crate::Index>, q: serde_json::Value| async move {
+ let req = xerj_query::parse_request(
+ &json!({"query": q, "size": 10, "track_total_hits": true}),
+ )
+ .unwrap();
+ let r = idx.search(&req).await.unwrap();
+ let mut ids: Vec<_> = r.hits.iter().map(|h| h.id.clone()).collect();
+ ids.sort();
+ (r.total.value, ids)
+ };
+ // must_not-ONLY [prefix "ap"] excludes apple(1); grape(2) + banana(3)
+ // must remain, pre AND post flush (before the fix, post-flush dropped
+ // ALL docs → (0, []) via the pure-negative FTS projection).
+ let want = (2u64, vec!["2".to_string(), "3".to_string()]);
+ for phase in ["pre", "post"] {
+ for q in [
+ json!({"bool": {"must_not": [{"prefix": {"k": "ap"}}]}}),
+ json!({"bool": {"must_not": [{"wildcard": {"k": "app*"}}]}}),
+ ] {
+ assert_eq!(
+ run(idx.clone(), q.clone()).await,
+ want,
+ "#797 {phase} {q}: must_not-only prefix/wildcard keeps non-matching docs"
+ );
+ }
+ if phase == "pre" {
+ idx.flush().await.unwrap();
+ assert_eq!(idx.memtable.doc_count(), 0);
+ }
+ }
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn mustnot_prefix_on_keyword_stays_case_sensitive_after_flush() {
// #794: a prefix/wildcard inside bool.must_not case-folded on the segment
@@ -42030,7 +42089,15 @@ fn query_node_to_fts_with_keyword_fields(
// — not projectable; fall back to the stored-doc scan.
Some(_) => return None,
}
- let mut projected_any = false;
+ // #797: a bool that projects NO positive (must/should/filter)
+ // clause must NOT become a pure-negative FTS bool. FTS (like
+ // Lucene) requires a positive clause to enumerate a base set, so a
+ // `must_not`-only projection matches NOTHING — whereas the true
+ // answer is "all docs minus the must_not matches". Track whether a
+ // positive clause projected; if none did, decline so the stored
+ // scan (whose base is all docs) answers, exactly as a `must_not`-
+ // only keyword `term` (which never projects) already does.
+ let mut projected_positive = false;
// CRITICAL: if a `must` child can't be projected to FTS we
// CANNOT lift just the projectable subset — the bool would
// become more permissive than the original query. Return
@@ -42044,7 +42111,7 @@ fn query_node_to_fts_with_keyword_fields(
keyword_fields,
)?;
bool_q = bool_q.must(fq);
- projected_any = true;
+ projected_positive = true;
}
// `filter` children constrain the hit set exactly like `must` but
// contribute NOTHING to `_score` (`BooleanClause.isScoring()` is
@@ -42075,7 +42142,7 @@ fn query_node_to_fts_with_keyword_fields(
keyword_fields,
) {
bool_q = bool_q.filter(fq);
- projected_any = true;
+ projected_positive = true;
}
}
for sub in should {
@@ -42091,7 +42158,7 @@ fn query_node_to_fts_with_keyword_fields(
keyword_fields,
)?;
bool_q = bool_q.should(fq);
- projected_any = true;
+ projected_positive = true;
}
// `must_not` children that don't project are similar: dropping
// a must_not relaxes the filter, which is wrong.
@@ -42107,10 +42174,9 @@ fn query_node_to_fts_with_keyword_fields(
keyword_fields,
) {
bool_q = bool_q.must_not(fq);
- projected_any = true;
}
}
- if !projected_any {
+ if !projected_positive {
return None;
}
Some(FtsQuery::Bool(Box::new(bool_q)))