AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): #797 must_not-only bool no longer projects to a pure-negative FTS bool (#798)
xerj-org0325552b1702
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex edde8ecbb..a7ce4e0b6 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -1659,6 +1659,65 @@ mod flush_publication_recovery_tests {}}+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn mustnot_only_prefix_keeps_nonmatching_docs_after_flush() {+ // #797: a bool with ONLY a must_not (no must/should/filter) containing a+ // prefix/wildcard leaf returns zero hits after flush — the segment+ // planner lets the negative leaf drive candidate generation, so the+ // non-matching docs never enter the set. `term` is unaffected. Base set+ // must be all docs, must_not subtracting only its matches.+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ let mut schema = Schema::empty();+ schema+ .add_field(FieldConfig::new("k", FieldType::Keyword))+ .unwrap();+ engine.create_index("mn797", schema).unwrap();+ let idx = engine.get_index("mn797").unwrap();+ idx.abort_background_tasks();+ // Data is case-fold-invariant for prefix "ap"/"app*": only "apple"+ // starts with it under ANY casing, so the correct answer does not+ // depend on the separate #794 keyword-prefix case-sensitivity fix.+ for (id, k) in [("1", "apple"), ("2", "grape"), ("3", "banana")] {+ idx.index_document(Some(id.into()), json!({ "k": k }))+ .await+ .unwrap();+ }+ let run = |idx: std::sync::Arc<crate::Index>, q: serde_json::Value| async move {+ let req = xerj_query::parse_request(+ &json!({"query": q, "size": 10, "track_total_hits": true}),+ )+ .unwrap();+ let r = idx.search(&req).await.unwrap();+ let mut ids: Vec<_> = r.hits.iter().map(|h| h.id.clone()).collect();+ ids.sort();+ (r.total.value, ids)+ };+ // must_not-ONLY [prefix "ap"] excludes apple(1); grape(2) + banana(3)+ // must remain, pre AND post flush (before the fix, post-flush dropped+ // ALL docs → (0, []) via the pure-negative FTS projection).+ let want = (2u64, vec!["2".to_string(), "3".to_string()]);+ for phase in ["pre", "post"] {+ for q in [+ json!({"bool": {"must_not": [{"prefix": {"k": "ap"}}]}}),+ json!({"bool": {"must_not": [{"wildcard": {"k": "app*"}}]}}),+ ] {+ assert_eq!(+ run(idx.clone(), q.clone()).await,+ want,+ "#797 {phase} {q}: must_not-only prefix/wildcard keeps non-matching docs"+ );+ }+ if phase == "pre" {+ idx.flush().await.unwrap();+ assert_eq!(idx.memtable.doc_count(), 0);+ }+ }+ }+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn mustnot_prefix_on_keyword_stays_case_sensitive_after_flush() {// #794: a prefix/wildcard inside bool.must_not case-folded on the segment@@ -42030,7 +42089,15 @@ fn query_node_to_fts_with_keyword_fields(// — not projectable; fall back to the stored-doc scan.Some(_) => return None,}- let mut projected_any = false;+ // #797: a bool that projects NO positive (must/should/filter)+ // clause must NOT become a pure-negative FTS bool. FTS (like+ // Lucene) requires a positive clause to enumerate a base set, so a+ // `must_not`-only projection matches NOTHING — whereas the true+ // answer is "all docs minus the must_not matches". Track whether a+ // positive clause projected; if none did, decline so the stored+ // scan (whose base is all docs) answers, exactly as a `must_not`-+ // only keyword `term` (which never projects) already does.+ let mut projected_positive = false;// CRITICAL: if a `must` child can't be projected to FTS we// CANNOT lift just the projectable subset — the bool would// become more permissive than the original query. Return@@ -42044,7 +42111,7 @@ fn query_node_to_fts_with_keyword_fields(keyword_fields,)?;bool_q = bool_q.must(fq);- projected_any = true;+ projected_positive = true;}// `filter` children constrain the hit set exactly like `must` but// contribute NOTHING to `_score` (`BooleanClause.isScoring()` is@@ -42075,7 +42142,7 @@ fn query_node_to_fts_with_keyword_fields(keyword_fields,) {bool_q = bool_q.filter(fq);- projected_any = true;+ projected_positive = true;}}for sub in should {@@ -42091,7 +42158,7 @@ fn query_node_to_fts_with_keyword_fields(keyword_fields,)?;bool_q = bool_q.should(fq);- projected_any = true;+ projected_positive = true;}// `must_not` children that don't project are similar: dropping// a must_not relaxes the filter, which is wrong.@@ -42107,10 +42174,9 @@ fn query_node_to_fts_with_keyword_fields(keyword_fields,) {bool_q = bool_q.must_not(fq);- projected_any = true;}}- if !projected_any {+ if !projected_positive {return None;}Some(FtsQuery::Bool(Box::new(bool_q)))