Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): bool with a non-projectable filter/must_not no longer diverts the scorer (#361)

xerj-orge6c31861a0fc
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 38ffe3e89..c851d26df 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -15892,6 +15892,19 @@ impl Index {
&kw_fields,
);
let needs_fts = fts_query_probe.is_some();
+ // #361: a bool with a non-projectable non-scoring (`filter`/`must_not`)
+ // child projects only its scoring subtree — the residual clause is
+ // absent from the FTS match set, so when FTS runs we must re-apply
+ // membership with `doc_matches_query` over the ORIGINAL query and
+ // switch off the seg_total count/bounded fast path (which assumes the
+ // FTS bool is the exact match set). Only meaningful when FTS runs.
+ let residual_gate = needs_fts
+ && bool_has_nonprojectable_nonscoring(
+ query,
+ &text_fields,
+ &exact_fields,
+ &kw_fields,
+ );
// A `query_string` whose projection DECLINED still has to be
// answered by the stored-doc scan — an over-cap `tokens × fields`
// cross-product returns `None`, and without the scan the segment
@@ -16195,7 +16208,7 @@ impl Index {
// skip past `materialisation_limit` matches while
// filling the page, so the page needs more than the
// top-`cap` by score.
- let fts_cap = if count_only || sort_topk.is_some() {
+ let fts_cap = if count_only || sort_topk.is_some() || residual_gate {
usize::MAX
} else {
// #179 — `deletes_present` no longer forces the
@@ -16247,6 +16260,122 @@ impl Index {
fts_handled = true;
// #361 — this segment's hits carry exact BM25.
fts_scored_applied = true;
+ // #361 — residual membership gate. This bool has a
+ // non-projectable, non-scoring `filter`/`must_not`
+ // that the projection dropped from the FTS bool, so
+ // `seg_hits` (with `fts_cap == usize::MAX` forced
+ // above) is a SUPERSET of the answer. Re-apply
+ // membership with `doc_matches_query` over the
+ // original (alias/keyword-resolved) query on every
+ // match, counting + materialising only survivors.
+ // A dedicated pass, not inline gating: the bounded
+ // walk below early-outs below `page_worst` and skips
+ // non-entering segments, so gating there would
+ // undercount. Cost is O(matches) source reads for
+ // residual queries only (uncommon) — the same
+ // correctness-over-speed convention as the deletes
+ // path.
+ if residual_gate {
+ let slices =
+ self.stored_slices_for(seg_id.as_str(), meta.doc_count);
+ let whole: Option<Vec<Value>> = if slices.is_some() {
+ None
+ } else {
+ let seg_reader = self.store.open_segment_arc(&seg_id)?;
+ match seg_reader.section(SectionType::Stored) {
+ Ok(Some(raw)) => {
+ xerj_storage::stored_codec::decode_stored(raw)
+ .ok()
+ .and_then(|b| {
+ serde_json::from_slice::<Vec<Value>>(&b)
+ .ok()
+ })
+ }
+ _ => None,
+ }
+ };
+ let fetch = |pos: u32| -> Option<Value> {
+ if let Some(s) = slices.as_deref() {
+ let &(a, b) = s.offsets.get(pos as usize)?;
+ serde_json::from_slice::<Value>(
+ s.bytes.get(a as usize..b as usize)?,
+ )
+ .ok()
+ } else {
+ whole
+ .as_ref()
+ .and_then(|d| d.get(pos as usize).cloned())
+ }
+ };
+ for sh in &seg_hits {
+ let doc = match fetch(sh.doc_id) {
+ Some(d) => d,
+ None => continue,
+ };
+ let id = doc
+ .get("_id")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
+ // Same liveness/staleness check the normal
+ // walk uses: skip tombstoned or superseded
+ // copies (live version in a newer segment).
+ let (mut hit_seq_no, mut hit_version) = (None, None);
+ if let Some(ver) = self.store.version_map.get(&id) {
+ if ver.deleted
+ || ver.segment_id.as_ref() != seg_id.as_str()
+ {
+ continue;
+ }
+ let (s, v) = self.hit_seq_version(&id, &ver);
+ hit_seq_no = s;
+ hit_version = v;
+ }
+ if seen_ids.contains(&id) {
+ continue;
+ }
+ let source =
+ doc.get("_source").cloned().unwrap_or(Value::Null);
+ // Membership: re-test the dropped residual
+ // clause via the whole original query.
+ let mut src_with_id = source.clone();
+ if let Some(obj) = src_with_id.as_object_mut() {
+ obj.insert(
+ "_id".to_string(),
+ Value::String(id.clone()),
+ );
+ }
+ if !doc_matches_query(query, &src_with_id) {
+ continue;
+ }
+ total_count += 1;
+ seen_ids.insert(id.clone());
+ let hit = Hit {
+ id,
+ score: sh.score,
+ source,
+ seq_no: hit_seq_no,
+ version: hit_version,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ if let Some(topk) = sort_topk.as_mut() {
+ let seq = hit_seq_no.unwrap_or(u64::MAX);
+ topk.offer(hit, seq, self);
+ } else {
+ all_hits.push(hit);
+ }
+ }
+ // Preserve the segment-tail deadline propagation
+ // that the `continue` below would otherwise skip.
+ if fts_deadline_tripped {
+ deadline_exceeded = true;
+ }
+ continue;
+ }
// Exact count — identical for count_only and size>0.
// GHOST WINDOW (b7 DEFECT 1c): `seg_total` is the
// segment's PHYSICAL match count — between a flush
@@ -39101,14 +39230,23 @@ fn query_node_to_fts_with_keyword_fields(
// `bool{must:[text], filter:[term]}` score the filter's BM25 on
// top of the text clause and reorder the page.
for sub in filter {
- let fq = query_node_to_fts_with_keyword_fields(
+ // #361: a non-projecting filter child (e.g. `exists`,
+ // non-numeric `range`) is a NON-SCORING residual — skip it
+ // (instead of `?`-aborting the whole projection) so the scoring
+ // must/should subtree keeps exact BM25. Because the residual is
+ // then absent from the FTS bool, membership is re-applied by the
+ // `residual_gate` pass in `search` (a `doc_matches_query` sweep
+ // over the original query; the shape is detected up-front by
+ // `bool_has_nonprojectable_nonscoring`).
+ if let Some(fq) = query_node_to_fts_with_keyword_fields(
sub,
text_fields,
exact_fields,
keyword_fields,
- )?;
- bool_q = bool_q.filter(fq);
- projected_any = true;
+ ) {
+ bool_q = bool_q.filter(fq);
+ projected_any = true;
+ }
}
for sub in should {
// A `should` clause that can't be projected to FTS (e.g.
@@ -39128,14 +39266,19 @@ fn query_node_to_fts_with_keyword_fields(
// `must_not` children that don't project are similar: dropping
// a must_not relaxes the filter, which is wrong.
for sub in must_not {
- let fq = query_node_to_fts_with_keyword_fields(
+ // #361: like `filter`, a non-projecting `must_not` child is a
+ // non-scoring residual — skip it (don't `?`-abort); the
+ // `residual_gate` pass in `search` re-excludes it via
+ // `doc_matches_query` over the original query.
+ if let Some(fq) = query_node_to_fts_with_keyword_fields(
sub,
text_fields,
exact_fields,
keyword_fields,
- )?;
- bool_q = bool_q.must_not(fq);
- projected_any = true;
+ ) {
+ bool_q = bool_q.must_not(fq);
+ projected_any = true;
+ }
}
if !projected_any {
return None;
@@ -39499,6 +39642,56 @@ fn query_node_to_fts_with_keyword_fields(
}
}
+/// #361: does `q` contain a `bool` whose `filter`/`must_not` child cannot be
+/// projected to FTS (e.g. `exists`, non-numeric `range`, `script`)?
+///
+/// Such a non-scoring child is skipped from the FTS bool by the projection
+/// (`query_node_to_fts_with_keyword_fields`) so the scoring subtree keeps exact
+/// BM25 — but it is then absent from the FTS match set, so membership must be
+/// re-applied by a `doc_matches_query` gate over the original query, and the
+/// count/materialisation fast path (which assumes the FTS bool IS the exact
+/// match set) must be switched off for the segment.
+fn bool_has_nonprojectable_nonscoring(
+ q: &QueryNode,
+ text_fields: &[String],
+ exact_fields: &std::collections::HashSet<String>,
+ keyword_fields: &std::collections::HashSet<String>,
+) -> bool {
+ match q {
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ ..
+ } => {
+ for sub in filter.iter().chain(must_not.iter()) {
+ if query_node_to_fts_with_keyword_fields(
+ sub,
+ text_fields,
+ exact_fields,
+ keyword_fields,
+ )
+ .is_none()
+ {
+ return true;
+ }
+ }
+ must.iter()
+ .chain(should.iter())
+ .chain(filter.iter())
+ .chain(must_not.iter())
+ .any(|c| {
+ bool_has_nonprojectable_nonscoring(c, text_fields, exact_fields, keyword_fields)
+ })
+ }
+ QueryNode::Constant { query, .. } => {
+ bool_has_nonprojectable_nonscoring(query, text_fields, exact_fields, keyword_fields)
+ }
+ _ => false,
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// FunctionScore helpers
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-engine/tests/bool_filter_is_non_scoring.rs b/engine/crates/xerj-engine/tests/bool_filter_is_non_scoring.rs
index 3cab90fff..3e88fa130 100644
--- a/engine/crates/xerj-engine/tests/bool_filter_is_non_scoring.rs
+++ b/engine/crates/xerj-engine/tests/bool_filter_is_non_scoring.rs
@@ -254,3 +254,210 @@ async fn filter_only_bool_still_scores_zero() {
);
}
}
+
+/// The still-live #361 case after #387: a **non-projectable** filter (here
+/// `exists`, which has no FTS projection arm) must not divert the MUST clause
+/// off the exact-BM25 scorer onto the IDF-less heuristic. `exists: repo`
+/// matches every doc, so this is a *no-op* filter — the headline of the issue
+/// ("a no-op filter flattens every score to a constant") — and the result
+/// must be byte-identical to the bare `match` reference.
+///
+/// Root cause: in `query_node_to_fts_with_keyword_fields` the Bool arm's
+/// filter loop does `let fq = ...(sub)?;`, and `exists` returns `None` (no FTS
+/// arm), so the `?` aborts the ENTIRE projection — dragging the scoring
+/// `must` subtree onto `score_query_against_doc` (`boost·(1+ln(1+tf))`, no
+/// IDF). #387 only covered *projectable* (`term`) filters.
+#[tokio::test]
+async fn nonprojectable_filter_does_not_divert_the_scorer() {
+ let (_dir, idx) = seed("bool-nonprojectable-filter").await;
+ let reference = idx.search(&req(text_query(), 200)).await.unwrap();
+ assert!(
+ score_map(&reference).values().any(|s| *s > 1.0),
+ "fixture must expose the exact BM25 path, got {:?}",
+ reference.hits.first().map(|h| h.score)
+ );
+
+ let filtered = idx
+ .search(&req(
+ json!({"bool": {
+ "must": [text_query()],
+ "filter": [{"exists": {"field": "repo"}}]
+ }}),
+ 50,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ filtered.total.value, 120,
+ "exists:repo matches every doc — a no-op filter"
+ );
+ assert_matches_reference(
+ "bool.must + non-projectable exists filter",
+ &reference,
+ &filtered,
+ );
+}
+
+/// The correctness half of the non-projectable-filter fix: a residual filter
+/// that genuinely EXCLUDES documents must still exclude them. Once the
+/// projection stops `?`-aborting on a non-projectable filter, the FTS bool no
+/// longer carries that filter — so membership must be re-applied by the
+/// caller's `doc_matches_query` gate, or the query over-returns.
+///
+/// Fixture: `tag` (keyword) is present on the 60 even docs only. `exists:tag`
+/// is non-projectable AND selective, so `bool{must:[match], filter:[exists:tag]}`
+/// must return exactly those 60 — not all 120.
+#[tokio::test]
+async fn residual_filter_still_excludes_nonmatching_docs() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("tag", FieldType::Keyword));
+ engine.create_index("residual-excl", schema).unwrap();
+ let idx = engine.get_index("residual-excl").unwrap();
+ for i in 0..120usize {
+ let alphas = "alpha ".repeat(1 + i % 5);
+ let mut doc = json!({ "body": alphas });
+ if i % 2 == 0 {
+ doc.as_object_mut()
+ .unwrap()
+ .insert("tag".into(), json!("kept"));
+ }
+ idx.index_document(Some(format!("d{i:03}")), doc)
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ let filtered = idx
+ .search(&req(
+ json!({"bool": {
+ "must": [{"match": {"body": "alpha"}}],
+ "filter": [{"exists": {"field": "tag"}}]
+ }}),
+ 200,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ filtered.total.value, 60,
+ "exists:tag is selective — only the 60 even docs carry `tag`; a missing \
+ membership gate over-returns all 120 (#361)"
+ );
+ for hit in &filtered.hits {
+ let n: usize = hit.id[1..].parse().unwrap();
+ assert_eq!(n % 2, 0, "{} has no `tag` and must be excluded", hit.id);
+ }
+}
+
+/// The residual gate must be page-size-invariant: forcing full materialisation
+/// and re-counting survivors must give the same top hit, the same score, and
+/// the same total at size:1 and size:200 (a gate that miscounts or mis-ranks
+/// would diverge).
+#[tokio::test]
+async fn residual_gate_is_page_size_invariant() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("tag", FieldType::Keyword));
+ engine.create_index("residual-page", schema).unwrap();
+ let idx = engine.get_index("residual-page").unwrap();
+ for i in 0..120usize {
+ let alphas = "alpha ".repeat(1 + i % 5);
+ let mut doc = json!({ "body": alphas });
+ if i % 2 == 0 {
+ doc.as_object_mut()
+ .unwrap()
+ .insert("tag".into(), json!("kept"));
+ }
+ idx.index_document(Some(format!("d{i:03}")), doc)
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ let q = json!({"bool": {
+ "must": [{"match": {"body": "alpha"}}],
+ "filter": [{"exists": {"field": "tag"}}]
+ }});
+ let deep = idx.search(&req(q.clone(), 200)).await.unwrap();
+ let shallow = idx.search(&req(q.clone(), 1)).await.unwrap();
+ assert_eq!(deep.total.value, 60, "residual total wrong at size:200");
+ assert_eq!(shallow.total.value, 60, "residual total wrong at size:1");
+ assert!(!deep.hits.is_empty() && !shallow.hits.is_empty());
+ assert_eq!(
+ shallow.hits[0].id, deep.hits[0].id,
+ "residual: size:1 and size:200 disagree on the best hit"
+ );
+ assert!(
+ (shallow.hits[0].score - deep.hits[0].score).abs() < 1e-4,
+ "residual: top hit scores {} at size:1 but {} at size:200",
+ shallow.hits[0].score,
+ deep.hits[0].score
+ );
+}
+
+/// A NON-projectable `must_not` (`exists`) is also skipped from the FTS bool by
+/// the projection and re-applied by the same gate — it must still EXCLUDE the
+/// docs it names (here: keep only the docs WITHOUT `tag`).
+#[tokio::test]
+async fn nonprojectable_must_not_still_excludes() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("tag", FieldType::Keyword));
+ engine.create_index("residual-mustnot", schema).unwrap();
+ let idx = engine.get_index("residual-mustnot").unwrap();
+ for i in 0..120usize {
+ let alphas = "alpha ".repeat(1 + i % 5);
+ let mut doc = json!({ "body": alphas });
+ if i % 2 == 0 {
+ doc.as_object_mut()
+ .unwrap()
+ .insert("tag".into(), json!("kept"));
+ }
+ idx.index_document(Some(format!("d{i:03}")), doc)
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ let excluded = idx
+ .search(&req(
+ json!({"bool": {
+ "must": [{"match": {"body": "alpha"}}],
+ "must_not": [{"exists": {"field": "tag"}}]
+ }}),
+ 200,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ excluded.total.value, 60,
+ "must_not exists:tag must exclude the 60 tagged docs, keeping the 60 untagged"
+ );
+ for hit in &excluded.hits {
+ let n: usize = hit.id[1..].parse().unwrap();
+ assert_eq!(
+ n % 2,
+ 1,
+ "{} HAS `tag` and must be excluded by must_not",
+ hit.id
+ );
+ }
+}