Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(query): multi_match must ignore unmapped fields, not zero the query (#217)

xerj-org437e475ea8df
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index edabbbbb1..27645c07e 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -31685,6 +31685,44 @@ fn prune_missing_should_fields(q: &FtsQuery, has_field: &dyn Fn(&str) -> bool) -
Some(FtsQuery::Bool(Box::new(pruned)))
}
+/// Is this `multi_match` field spec (boost already stripped) a MAPPED field
+/// for projection purposes?
+///
+/// ES resolves each `multi_match` field against the mapping and silently
+/// skips the ones that do not resolve; XERJ's equivalent of "resolves" is:
+///
+/// * a schema field itself (`text_fields` ∪ `exact_fields` is exactly the
+/// flat schema field list — see the construction in `search`), or
+/// * a dotted path whose prefix is a schema field. Sub-fields carry no
+/// top-level FTS side-car of their own (`build_fts_field_configs` and
+/// the memtable indexer both key on top-level `_source` keys), so both
+/// multi-fields (`title.keyword` — shares the parent's source value)
+/// and object sub-paths (`user.name`) are served by the stored-doc
+/// scan's `get_field_value` fallback today. Keeping their clause in
+/// the projection is what routes the query onto that scan (the
+/// per-segment `fts_has_field` gate fails), so dropping them here would
+/// change matching for MAPPED data — they must survive the filter.
+///
+/// Only a spec with NO schema field anywhere on its dotted prefix chain is
+/// unmapped in ES's sense, and those are exactly the ones ES ignores.
+fn multi_match_field_is_mapped(
+ field: &str,
+ text_fields: &[String],
+ exact_fields: &std::collections::HashSet<String>,
+) -> bool {
+ let known = |f: &str| exact_fields.contains(f) || text_fields.iter().any(|t| t == f);
+ if known(field) {
+ return true;
+ }
+ // Walk every dot-prefix: `a.b.c` is mapped when `a` or `a.b` is.
+ for (i, ch) in field.char_indices() {
+ if ch == '.' && known(&field[..i]) {
+ return true;
+ }
+ }
+ false
+}
+
/// Collect every field name referenced by an FTS query tree.
///
/// Used to (a) extend the set of side-car fields `FtsIndexReader::open`
@@ -31980,6 +32018,7 @@ fn query_node_to_fts(
query,
fields,
match_type,
+ operator,
boost,
..
} => {
@@ -31987,6 +32026,22 @@ fn query_node_to_fts(
let analyzer = registry.get_analyzer("standard")?;
let tokens = analyzer.analyze(query);
// Split boost factors out of field specs (e.g. "title^3" → ("title", 3.0)).
+ //
+ // ES IGNORES unmapped fields in `multi_match` (its per-type
+ // builders skip any field the mapping cannot resolve): the query
+ // runs over the mapped subset, and only when EVERY field is
+ // unmapped does it match nothing. Before this filter, a clause
+ // was built for the unmapped field anyway; the per-segment
+ // "reader has every queried field" gate then refused FTS for the
+ // WHOLE query and the stored-doc fallback — whose multi_match
+ // default arm requires the entire query string as one substring —
+ // silently zeroed every multi-token query (#217; single-token
+ // queries survived because the one token IS the whole string).
+ // A field spec is kept when it names a schema field OR a dotted
+ // sub-path under one (`title.keyword`, `user.name`): those have
+ // no top-level FTS side-car and must keep today's stored-scan
+ // routing, which resolves them via the multi-field/_source
+ // fallback in `get_field_value`.
let field_specs: Vec<(String, f32)> = if fields.is_empty() {
text_fields.iter().map(|s| (s.clone(), 1.0)).collect()
} else {
@@ -31996,11 +32051,34 @@ fn query_node_to_fts(
let (f, b) = parse_field_boost(s);
(f.to_string(), b)
})
+ .filter(|(f, _)| multi_match_field_is_mapped(f, text_fields, exact_fields))
.collect()
};
+ // Every named field is unmapped: ES matches NOTHING (no error).
+ // Returning None keeps `needs_fts` false and routes the request
+ // to the stored-doc scan (`is_doc_scan_query` includes
+ // MultiMatch), which finds no such fields in any document.
if field_specs.is_empty() {
return None;
}
+ // `operator: and` binds PER FIELD for the field-centric types
+ // (ES best_fields/most_fields build one match query per field
+ // with the given operator): every token must match in the SAME
+ // field — mirror the Match arm's AND lowering (must-clauses).
+ // `cross_fields` is term-centric (each token must appear in at
+ // least ONE of the fields, not all in the same one), which a
+ // per-field FTS bool cannot express — decline and leave it to
+ // the stored-doc scan, whose cross_fields arm implements the
+ // combined-text semantics. Without this, dropping an unmapped
+ // field would FLIP an `operator: and` query from the scan's
+ // AND semantics onto the FTS OR path, over-matching.
+ let is_and = matches!(operator, Some(xerj_query::ast::BoolOperator::And));
+ if is_and
+ && tokens.len() > 1
+ && matches!(match_type, xerj_query::ast::MultiMatchType::CrossFields)
+ {
+ return None;
+ }
// Keyword-typed fields match by WHOLE value (their FST holds one
// case-preserved keyword-analyzer token per value), so an empty
// standard-token set only kills the projection when no keyword
@@ -32039,10 +32117,16 @@ fn query_node_to_fts(
*fb,
)));
} else {
+ // `operator: and` → per-field must (all tokens in this
+ // field), same as the Match arm; default → per-field OR.
let mut field_bool = FtsBool::new().boost(*fb);
for token in &tokens {
- field_bool = field_bool
- .should(FtsQuery::Term(FtsTerm::new(field.as_str(), &token.text)));
+ let term = FtsQuery::Term(FtsTerm::new(field.as_str(), &token.text));
+ field_bool = if is_and {
+ field_bool.must(term)
+ } else {
+ field_bool.should(term)
+ };
}
per_field.push(FtsQuery::Bool(Box::new(field_bool)));
}
@@ -34670,6 +34754,164 @@ mod fts_projection_tests {
}
}
+ fn mm(fields: &[&str], query: &str) -> QueryNode {
+ QueryNode::MultiMatch {
+ fields: fields.iter().map(|s| s.to_string()).collect(),
+ query: query.into(),
+ match_type: xerj_query::ast::MultiMatchType::BestFields,
+ operator: None,
+ analyzer: None,
+ boost: None,
+ }
+ }
+
+ /// #217 — an UNMAPPED field in `multi_match` must be dropped from the
+ /// projection (ES ignores unmapped fields), so the surviving clause is
+ /// byte-identical to the same query without that field. Before the fix
+ /// the ghost clause survived, the per-segment `fts_has_field` gate then
+ /// refused FTS for the whole query, and the stored-doc fallback's
+ /// whole-substring semantics zeroed every multi-token query.
+ #[test]
+ fn multi_match_unmapped_field_is_dropped_from_the_projection() {
+ let text_fields = vec!["body".to_string()];
+ let with_ghost = query_node_to_fts(
+ &mm(&["body", "ghost"], "merge zzzznotpresent"),
+ &text_fields,
+ &kw(&[]),
+ )
+ .expect("projects over the mapped subset");
+ let without = query_node_to_fts(
+ &mm(&["body"], "merge zzzznotpresent"),
+ &text_fields,
+ &kw(&[]),
+ )
+ .expect("control projects");
+ assert_eq!(
+ format!("{with_ghost:?}"),
+ format!("{without:?}"),
+ "the unmapped field must not change the projection"
+ );
+ // And no clause may reference the ghost field at all.
+ let mut fields = Vec::new();
+ collect_fts_query_fields(&with_ghost, &mut fields);
+ assert_eq!(fields, vec!["body".to_string()]);
+ }
+
+ /// #217 — single-token shape, same rule (this shape happened to survive
+ /// the bug via the stored scan; post-fix it stays on the postings path).
+ #[test]
+ fn multi_match_unmapped_field_single_token_projects_mapped_term() {
+ let fq = query_node_to_fts(
+ &mm(&["body", "ghost"], "merge"),
+ &["body".to_string()],
+ &kw(&[]),
+ )
+ .expect("projects");
+ match fq {
+ FtsQuery::Term(t) => {
+ assert_eq!(t.field, "body");
+ assert_eq!(t.term, "merge");
+ }
+ other => panic!("expected bare term over the mapped field, got {other:?}"),
+ }
+ }
+
+ /// #217 — EVERY named field unmapped: ES matches nothing (no error).
+ /// The projection declines; `is_doc_scan_query` routes the request to
+ /// the stored-doc scan, which finds no such fields in any document.
+ #[test]
+ fn multi_match_all_fields_unmapped_projects_none() {
+ assert!(query_node_to_fts(
+ &mm(&["ghost1", "ghost2"], "merge zzzznotpresent"),
+ &["body".to_string()],
+ &kw(&["status"]),
+ )
+ .is_none());
+ }
+
+ /// Dotted sub-paths under a mapped parent (`title.keyword`, `user.name`)
+ /// are NOT unmapped — they have no top-level FTS side-car and are served
+ /// by the stored scan's multi-field/_source fallback, so their clause
+ /// must SURVIVE the filter to keep that routing.
+ #[test]
+ fn multi_match_dotted_subfield_of_mapped_parent_is_kept() {
+ let fq = query_node_to_fts(
+ &mm(&["body", "title.keyword"], "merge policy"),
+ &["body".to_string(), "title".to_string()],
+ &kw(&[]),
+ )
+ .expect("projects");
+ let mut fields = Vec::new();
+ collect_fts_query_fields(&fq, &mut fields);
+ assert!(
+ fields.contains(&"title.keyword".to_string()),
+ "sub-field clause dropped: {fields:?}"
+ );
+ // …while a dotted path with an UNMAPPED root is dropped.
+ let fq = query_node_to_fts(
+ &mm(&["body", "ghost.sub"], "merge policy"),
+ &["body".to_string()],
+ &kw(&[]),
+ )
+ .expect("projects");
+ let mut fields = Vec::new();
+ collect_fts_query_fields(&fq, &mut fields);
+ assert_eq!(fields, vec!["body".to_string()]);
+ }
+
+ /// `operator: and` on the field-centric types binds per field: the
+ /// projection must demand every token in the SAME field (must-clauses),
+ /// mirroring the Match arm — not the OR-bool the default shape uses.
+ /// Without this, dropping an unmapped field would flip an AND query
+ /// from the scan's AND semantics onto the FTS OR path and over-match.
+ #[test]
+ fn multi_match_operator_and_projects_per_field_must() {
+ let q = QueryNode::MultiMatch {
+ fields: vec!["body".into(), "title".into()],
+ query: "merge policy".into(),
+ match_type: xerj_query::ast::MultiMatchType::BestFields,
+ operator: Some(xerj_query::ast::BoolOperator::And),
+ analyzer: None,
+ boost: None,
+ };
+ let fq = query_node_to_fts(&q, &["body".to_string(), "title".to_string()], &kw(&[]))
+ .expect("projects");
+ match fq {
+ FtsQuery::DisMax(d) => {
+ assert_eq!(d.queries.len(), 2);
+ for clause in &d.queries {
+ match clause {
+ FtsQuery::Bool(b) => {
+ assert_eq!(b.must.len(), 2, "every token required per field");
+ assert!(b.should.is_empty());
+ }
+ other => panic!("expected per-field must-bool, got {other:?}"),
+ }
+ }
+ }
+ other => panic!("expected dis_max, got {other:?}"),
+ }
+ }
+
+ /// `cross_fields` + `operator: and` is term-centric (each token must
+ /// appear in at least ONE field) — inexpressible as a per-field bool, so
+ /// the projection declines and the stored-doc scan's cross_fields arm
+ /// keeps the combined-text semantics.
+ #[test]
+ fn multi_match_cross_fields_and_declines_projection() {
+ let q = QueryNode::MultiMatch {
+ fields: vec!["body".into(), "title".into()],
+ query: "merge policy".into(),
+ match_type: xerj_query::ast::MultiMatchType::CrossFields,
+ operator: Some(xerj_query::ast::BoolOperator::And),
+ analyzer: None,
+ boost: None,
+ };
+ assert!(
+ query_node_to_fts(&q, &["body".to_string(), "title".to_string()], &kw(&[]),).is_none()
+ );
+ }
+
/// match_phrase on a KEYWORD field projects to a single whole-value term
/// (routed through the postings index instead of the O(N) brute scan that
/// made keyword match_phrase multi-second). ES: match_phrase on keyword is
diff --git a/engine/crates/xerj-engine/tests/multi_match_unmapped_field.rs b/engine/crates/xerj-engine/tests/multi_match_unmapped_field.rs
new file mode 100644
index 000000000..bef02eaa0
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/multi_match_unmapped_field.rs
@@ -0,0 +1,205 @@
+//! Regression tests for #217 — `multi_match` naming an UNMAPPED field.
+//!
+//! ES ignores unmapped fields in `multi_match`: the query runs over the
+//! mapped subset, and only when EVERY field is unmapped does it match
+//! nothing (no error either way). XERJ instead built an FTS clause for the
+//! unmapped field; the per-segment "reader has every queried field" gate
+//! then refused the postings path for the WHOLE query, and the stored-doc
+//! fallback — whose default multi_match arm requires the entire query
+//! string as one contiguous substring — silently zeroed every multi-token
+//! query. Single-token queries survived because the one token IS the whole
+//! string, which is what kept the bug invisible to the ES-YAML suite.
+//!
+//! Every search here runs AFTER a flush: the memtable path has a separate,
+//! still-open multi-token divergence (MULTIMATCH_DEFECT.md, defect 2) that
+//! these tests deliberately do not encode expectations for.
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn req(q: Value) -> xerj_query::ast::SearchRequest {
+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")
+}
+
+/// One flushed doc with two mapped text fields (`body`, `title`); `ghost`
+/// never appears anywhere, so dynamic mapping never creates it.
+async fn seed(engine: &Engine, name: &str) -> std::sync::Arc<xerj_engine::Index> {
+ engine.create_index(name, Schema::empty()).unwrap();
+ let idx = engine.get_index(name).unwrap();
+ idx.index_document(
+ Some("1".to_string()),
+ json!({
+ "body": "the log merge policy groups segments into size buckets",
+ "title": "merge policy"
+ }),
+ )
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+ idx
+}
+
+/// #217 headline: an unmapped field must not change the result of a
+/// multi-token query — same hits as the identical query without it.
+#[tokio::test]
+async fn unmapped_field_must_not_zero_a_multi_token_multi_match() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mm217").await;
+
+ let control = idx
+ .search(&req(json!({"multi_match": {
+ "query": "merge zzzznotpresent", "fields": ["body"]}})))
+ .await
+ .unwrap();
+ assert_eq!(control.total.value, 1, "control: one token present in body");
+
+ let with_ghost = idx
+ .search(&req(json!({"multi_match": {
+ "query": "merge zzzznotpresent", "fields": ["body", "ghost"]}})))
+ .await
+ .unwrap();
+ assert_eq!(
+ with_ghost.total.value, 1,
+ "an unmapped field removed a document the mapped subset matches (#217)"
+ );
+ assert_eq!(
+ control.hits[0].id, with_ghost.hits[0].id,
+ "same document either way"
+ );
+}
+
+/// Single-token shape with the same unmapped field — the case that already
+/// worked (via the stored scan) and must keep working on the postings path.
+#[tokio::test]
+async fn unmapped_field_single_token_still_matches() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mm217_single").await;
+
+ let r = idx
+ .search(&req(json!({"multi_match": {
+ "query": "merge", "fields": ["body", "ghost"]}})))
+ .await
+ .unwrap();
+ assert_eq!(r.total.value, 1);
+}
+
+/// EVERY named field unmapped: ES semantics — match nothing, no error.
+#[tokio::test]
+async fn all_fields_unmapped_matches_nothing_without_error() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mm217_allghost").await;
+
+ for query in ["merge zzzznotpresent", "merge"] {
+ let r = idx
+ .search(&req(json!({"multi_match": {
+ "query": query, "fields": ["ghost1", "ghost2"]}})))
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, 0,
+ "query {query:?} over only unmapped fields"
+ );
+ }
+}
+
+/// The same zeroing hit every field-centric type through the shared
+/// lowering; `most_fields` is the non-dis_max shape.
+#[tokio::test]
+async fn most_fields_multi_token_with_unmapped_field() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mm217_most").await;
+
+ let r = idx
+ .search(&req(json!({"multi_match": {
+ "query": "merge zzzznotpresent",
+ "fields": ["body", "ghost"],
+ "type": "most_fields"}})))
+ .await
+ .unwrap();
+ assert_eq!(r.total.value, 1, "most_fields shares the lowering (#217)");
+}
+
+/// `operator: and` binds PER FIELD (ES best_fields/most_fields build one
+/// match query per field with the operator): a document matches only when
+/// ONE field holds every token. The pre-fix FTS lowering ignored the
+/// operator and OR'd the tokens, over-matching whenever every named field
+/// was mapped.
+#[tokio::test]
+async fn operator_and_requires_every_token_in_one_field() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mm217_and").await;
+
+ // No single field holds both "merge" and "zzzznotpresent" → 0 hits.
+ let r = idx
+ .search(&req(json!({"multi_match": {
+ "query": "merge zzzznotpresent",
+ "fields": ["body", "title"],
+ "operator": "and"}})))
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, 0,
+ "operator:and must not degrade to OR over tokens"
+ );
+
+ // `body` holds both tokens; the unmapped field must not change that —
+ // and in particular must not flip the query from AND onto an OR path.
+ let r = idx
+ .search(&req(json!({"multi_match": {
+ "query": "merge policy",
+ "fields": ["body", "ghost"],
+ "operator": "and"}})))
+ .await
+ .unwrap();
+ assert_eq!(r.total.value, 1, "mapped subset satisfies per-field AND");
+}
+
+/// `cross_fields` + `operator: and` is term-centric — every token must
+/// appear in at least ONE of the (mapped) fields, NOT all in the same one.
+/// Here `bravo` lives only in `alpha` and `charlie` only in `beta`, so a
+/// per-field AND would find nothing while the combined view matches — with
+/// or without an unmapped field in the list.
+#[tokio::test]
+async fn cross_fields_and_matches_across_fields() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ engine.create_index("mm217_cross", Schema::empty()).unwrap();
+ let idx = engine.get_index("mm217_cross").unwrap();
+ idx.index_document(
+ Some("1".to_string()),
+ json!({ "alpha": "delta bravo", "beta": "charlie echo" }),
+ )
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+
+ for fields in [json!(["alpha", "beta"]), json!(["alpha", "beta", "ghost"])] {
+ let r = idx
+ .search(&req(json!({"multi_match": {
+ "query": "bravo charlie",
+ "fields": fields,
+ "type": "cross_fields",
+ "operator": "and"}})))
+ .await
+ .unwrap();
+ assert_eq!(
+ r.total.value, 1,
+ "combined-text AND across mapped fields ({fields})"
+ );
+ }
+}