AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(query): multi_match must ignore unmapped fields, not zero the query (#217)
xerj-org437e475ea8df
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex edabbbbb1..27645c07e 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -31685,6 +31685,44 @@ fn prune_missing_should_fields(q: &FtsQuery, has_field: &dyn Fn(&str) -> bool) -Some(FtsQuery::Bool(Box::new(pruned)))}+/// Is this `multi_match` field spec (boost already stripped) a MAPPED field+/// for projection purposes?+///+/// ES resolves each `multi_match` field against the mapping and silently+/// skips the ones that do not resolve; XERJ's equivalent of "resolves" is:+///+/// * a schema field itself (`text_fields` ∪ `exact_fields` is exactly the+/// flat schema field list — see the construction in `search`), or+/// * a dotted path whose prefix is a schema field. Sub-fields carry no+/// top-level FTS side-car of their own (`build_fts_field_configs` and+/// the memtable indexer both key on top-level `_source` keys), so both+/// multi-fields (`title.keyword` — shares the parent's source value)+/// and object sub-paths (`user.name`) are served by the stored-doc+/// scan's `get_field_value` fallback today. Keeping their clause in+/// the projection is what routes the query onto that scan (the+/// per-segment `fts_has_field` gate fails), so dropping them here would+/// change matching for MAPPED data — they must survive the filter.+///+/// Only a spec with NO schema field anywhere on its dotted prefix chain is+/// unmapped in ES's sense, and those are exactly the ones ES ignores.+fn multi_match_field_is_mapped(+ field: &str,+ text_fields: &[String],+ exact_fields: &std::collections::HashSet<String>,+) -> bool {+ let known = |f: &str| exact_fields.contains(f) || text_fields.iter().any(|t| t == f);+ if known(field) {+ return true;+ }+ // Walk every dot-prefix: `a.b.c` is mapped when `a` or `a.b` is.+ for (i, ch) in field.char_indices() {+ if ch == '.' && known(&field[..i]) {+ return true;+ }+ }+ false+}+/// Collect every field name referenced by an FTS query tree.////// Used to (a) extend the set of side-car fields `FtsIndexReader::open`@@ -31980,6 +32018,7 @@ fn query_node_to_fts(query,fields,match_type,+ operator,boost,..} => {@@ -31987,6 +32026,22 @@ fn query_node_to_fts(let analyzer = registry.get_analyzer("standard")?;let tokens = analyzer.analyze(query);// Split boost factors out of field specs (e.g. "title^3" → ("title", 3.0)).+ //+ // ES IGNORES unmapped fields in `multi_match` (its per-type+ // builders skip any field the mapping cannot resolve): the query+ // runs over the mapped subset, and only when EVERY field is+ // unmapped does it match nothing. Before this filter, a clause+ // was built for the unmapped field anyway; the per-segment+ // "reader has every queried field" gate then refused FTS for the+ // WHOLE query and the stored-doc fallback — whose multi_match+ // default arm requires the entire query string as one substring —+ // silently zeroed every multi-token query (#217; single-token+ // queries survived because the one token IS the whole string).+ // A field spec is kept when it names a schema field OR a dotted+ // sub-path under one (`title.keyword`, `user.name`): those have+ // no top-level FTS side-car and must keep today's stored-scan+ // routing, which resolves them via the multi-field/_source+ // fallback in `get_field_value`.let field_specs: Vec<(String, f32)> = if fields.is_empty() {text_fields.iter().map(|s| (s.clone(), 1.0)).collect()} else {@@ -31996,11 +32051,34 @@ fn query_node_to_fts(let (f, b) = parse_field_boost(s);(f.to_string(), b)})+ .filter(|(f, _)| multi_match_field_is_mapped(f, text_fields, exact_fields)).collect()};+ // Every named field is unmapped: ES matches NOTHING (no error).+ // Returning None keeps `needs_fts` false and routes the request+ // to the stored-doc scan (`is_doc_scan_query` includes+ // MultiMatch), which finds no such fields in any document.if field_specs.is_empty() {return None;}+ // `operator: and` binds PER FIELD for the field-centric types+ // (ES best_fields/most_fields build one match query per field+ // with the given operator): every token must match in the SAME+ // field — mirror the Match arm's AND lowering (must-clauses).+ // `cross_fields` is term-centric (each token must appear in at+ // least ONE of the fields, not all in the same one), which a+ // per-field FTS bool cannot express — decline and leave it to+ // the stored-doc scan, whose cross_fields arm implements the+ // combined-text semantics. Without this, dropping an unmapped+ // field would FLIP an `operator: and` query from the scan's+ // AND semantics onto the FTS OR path, over-matching.+ let is_and = matches!(operator, Some(xerj_query::ast::BoolOperator::And));+ if is_and+ && tokens.len() > 1+ && matches!(match_type, xerj_query::ast::MultiMatchType::CrossFields)+ {+ return None;+ }// Keyword-typed fields match by WHOLE value (their FST holds one// case-preserved keyword-analyzer token per value), so an empty// standard-token set only kills the projection when no keyword@@ -32039,10 +32117,16 @@ fn query_node_to_fts(*fb,)));} else {+ // `operator: and` → per-field must (all tokens in this+ // field), same as the Match arm; default → per-field OR.let mut field_bool = FtsBool::new().boost(*fb);for token in &tokens {- field_bool = field_bool- .should(FtsQuery::Term(FtsTerm::new(field.as_str(), &token.text)));+ let term = FtsQuery::Term(FtsTerm::new(field.as_str(), &token.text));+ field_bool = if is_and {+ field_bool.must(term)+ } else {+ field_bool.should(term)+ };}per_field.push(FtsQuery::Bool(Box::new(field_bool)));}@@ -34670,6 +34754,164 @@ mod fts_projection_tests {}}+ fn mm(fields: &[&str], query: &str) -> QueryNode {+ QueryNode::MultiMatch {+ fields: fields.iter().map(|s| s.to_string()).collect(),+ query: query.into(),+ match_type: xerj_query::ast::MultiMatchType::BestFields,+ operator: None,+ analyzer: None,+ boost: None,+ }+ }++ /// #217 — an UNMAPPED field in `multi_match` must be dropped from the+ /// projection (ES ignores unmapped fields), so the surviving clause is+ /// byte-identical to the same query without that field. Before the fix+ /// the ghost clause survived, the per-segment `fts_has_field` gate then+ /// refused FTS for the whole query, and the stored-doc fallback's+ /// whole-substring semantics zeroed every multi-token query.+ #[test]+ fn multi_match_unmapped_field_is_dropped_from_the_projection() {+ let text_fields = vec!["body".to_string()];+ let with_ghost = query_node_to_fts(+ &mm(&["body", "ghost"], "merge zzzznotpresent"),+ &text_fields,+ &kw(&[]),+ )+ .expect("projects over the mapped subset");+ let without = query_node_to_fts(+ &mm(&["body"], "merge zzzznotpresent"),+ &text_fields,+ &kw(&[]),+ )+ .expect("control projects");+ assert_eq!(+ format!("{with_ghost:?}"),+ format!("{without:?}"),+ "the unmapped field must not change the projection"+ );+ // And no clause may reference the ghost field at all.+ let mut fields = Vec::new();+ collect_fts_query_fields(&with_ghost, &mut fields);+ assert_eq!(fields, vec!["body".to_string()]);+ }++ /// #217 — single-token shape, same rule (this shape happened to survive+ /// the bug via the stored scan; post-fix it stays on the postings path).+ #[test]+ fn multi_match_unmapped_field_single_token_projects_mapped_term() {+ let fq = query_node_to_fts(+ &mm(&["body", "ghost"], "merge"),+ &["body".to_string()],+ &kw(&[]),+ )+ .expect("projects");+ match fq {+ FtsQuery::Term(t) => {+ assert_eq!(t.field, "body");+ assert_eq!(t.term, "merge");+ }+ other => panic!("expected bare term over the mapped field, got {other:?}"),+ }+ }++ /// #217 — EVERY named field unmapped: ES matches nothing (no error).+ /// The projection declines; `is_doc_scan_query` routes the request to+ /// the stored-doc scan, which finds no such fields in any document.+ #[test]+ fn multi_match_all_fields_unmapped_projects_none() {+ assert!(query_node_to_fts(+ &mm(&["ghost1", "ghost2"], "merge zzzznotpresent"),+ &["body".to_string()],+ &kw(&["status"]),+ )+ .is_none());+ }++ /// Dotted sub-paths under a mapped parent (`title.keyword`, `user.name`)+ /// are NOT unmapped — they have no top-level FTS side-car and are served+ /// by the stored scan's multi-field/_source fallback, so their clause+ /// must SURVIVE the filter to keep that routing.+ #[test]+ fn multi_match_dotted_subfield_of_mapped_parent_is_kept() {+ let fq = query_node_to_fts(+ &mm(&["body", "title.keyword"], "merge policy"),+ &["body".to_string(), "title".to_string()],+ &kw(&[]),+ )+ .expect("projects");+ let mut fields = Vec::new();+ collect_fts_query_fields(&fq, &mut fields);+ assert!(+ fields.contains(&"title.keyword".to_string()),+ "sub-field clause dropped: {fields:?}"+ );+ // …while a dotted path with an UNMAPPED root is dropped.+ let fq = query_node_to_fts(+ &mm(&["body", "ghost.sub"], "merge policy"),+ &["body".to_string()],+ &kw(&[]),+ )+ .expect("projects");+ let mut fields = Vec::new();+ collect_fts_query_fields(&fq, &mut fields);+ assert_eq!(fields, vec!["body".to_string()]);+ }++ /// `operator: and` on the field-centric types binds per field: the+ /// projection must demand every token in the SAME field (must-clauses),+ /// mirroring the Match arm — not the OR-bool the default shape uses.+ /// Without this, dropping an unmapped field would flip an AND query+ /// from the scan's AND semantics onto the FTS OR path and over-match.+ #[test]+ fn multi_match_operator_and_projects_per_field_must() {+ let q = QueryNode::MultiMatch {+ fields: vec!["body".into(), "title".into()],+ query: "merge policy".into(),+ match_type: xerj_query::ast::MultiMatchType::BestFields,+ operator: Some(xerj_query::ast::BoolOperator::And),+ analyzer: None,+ boost: None,+ };+ let fq = query_node_to_fts(&q, &["body".to_string(), "title".to_string()], &kw(&[]))+ .expect("projects");+ match fq {+ FtsQuery::DisMax(d) => {+ assert_eq!(d.queries.len(), 2);+ for clause in &d.queries {+ match clause {+ FtsQuery::Bool(b) => {+ assert_eq!(b.must.len(), 2, "every token required per field");+ assert!(b.should.is_empty());+ }+ other => panic!("expected per-field must-bool, got {other:?}"),+ }+ }+ }+ other => panic!("expected dis_max, got {other:?}"),+ }+ }++ /// `cross_fields` + `operator: and` is term-centric (each token must+ /// appear in at least ONE field) — inexpressible as a per-field bool, so+ /// the projection declines and the stored-doc scan's cross_fields arm+ /// keeps the combined-text semantics.+ #[test]+ fn multi_match_cross_fields_and_declines_projection() {+ let q = QueryNode::MultiMatch {+ fields: vec!["body".into(), "title".into()],+ query: "merge policy".into(),+ match_type: xerj_query::ast::MultiMatchType::CrossFields,+ operator: Some(xerj_query::ast::BoolOperator::And),+ analyzer: None,+ boost: None,+ };+ assert!(+ query_node_to_fts(&q, &["body".to_string(), "title".to_string()], &kw(&[]),).is_none()+ );+ }+/// match_phrase on a KEYWORD field projects to a single whole-value term/// (routed through the postings index instead of the O(N) brute scan that/// made keyword match_phrase multi-second). ES: match_phrase on keyword isdiff --git a/engine/crates/xerj-engine/tests/multi_match_unmapped_field.rs b/engine/crates/xerj-engine/tests/multi_match_unmapped_field.rsnew file mode 100644index 000000000..bef02eaa0--- /dev/null+++ b/engine/crates/xerj-engine/tests/multi_match_unmapped_field.rs@@ -0,0 +1,205 @@+//! Regression tests for #217 — `multi_match` naming an UNMAPPED field.+//!+//! ES ignores unmapped fields in `multi_match`: the query runs over the+//! mapped subset, and only when EVERY field is unmapped does it match+//! nothing (no error either way). XERJ instead built an FTS clause for the+//! unmapped field; the per-segment "reader has every queried field" gate+//! then refused the postings path for the WHOLE query, and the stored-doc+//! fallback — whose default multi_match arm requires the entire query+//! string as one contiguous substring — silently zeroed every multi-token+//! query. Single-token queries survived because the one token IS the whole+//! string, which is what kept the bug invisible to the ES-YAML suite.+//!+//! Every search here runs AFTER a flush: the memtable path has a separate,+//! still-open multi-token divergence (MULTIMATCH_DEFECT.md, defect 2) that+//! these tests deliberately do not encode expectations for.++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;+use xerj_engine::Engine;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn req(q: Value) -> xerj_query::ast::SearchRequest {+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")+}++/// One flushed doc with two mapped text fields (`body`, `title`); `ghost`+/// never appears anywhere, so dynamic mapping never creates it.+async fn seed(engine: &Engine, name: &str) -> std::sync::Arc<xerj_engine::Index> {+ engine.create_index(name, Schema::empty()).unwrap();+ let idx = engine.get_index(name).unwrap();+ idx.index_document(+ Some("1".to_string()),+ json!({+ "body": "the log merge policy groups segments into size buckets",+ "title": "merge policy"+ }),+ )+ .await+ .unwrap();+ idx.flush().await.unwrap();+ idx+}++/// #217 headline: an unmapped field must not change the result of a+/// multi-token query — same hits as the identical query without it.+#[tokio::test]+async fn unmapped_field_must_not_zero_a_multi_token_multi_match() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mm217").await;++ let control = idx+ .search(&req(json!({"multi_match": {+ "query": "merge zzzznotpresent", "fields": ["body"]}})))+ .await+ .unwrap();+ assert_eq!(control.total.value, 1, "control: one token present in body");++ let with_ghost = idx+ .search(&req(json!({"multi_match": {+ "query": "merge zzzznotpresent", "fields": ["body", "ghost"]}})))+ .await+ .unwrap();+ assert_eq!(+ with_ghost.total.value, 1,+ "an unmapped field removed a document the mapped subset matches (#217)"+ );+ assert_eq!(+ control.hits[0].id, with_ghost.hits[0].id,+ "same document either way"+ );+}++/// Single-token shape with the same unmapped field — the case that already+/// worked (via the stored scan) and must keep working on the postings path.+#[tokio::test]+async fn unmapped_field_single_token_still_matches() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mm217_single").await;++ let r = idx+ .search(&req(json!({"multi_match": {+ "query": "merge", "fields": ["body", "ghost"]}})))+ .await+ .unwrap();+ assert_eq!(r.total.value, 1);+}++/// EVERY named field unmapped: ES semantics — match nothing, no error.+#[tokio::test]+async fn all_fields_unmapped_matches_nothing_without_error() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mm217_allghost").await;++ for query in ["merge zzzznotpresent", "merge"] {+ let r = idx+ .search(&req(json!({"multi_match": {+ "query": query, "fields": ["ghost1", "ghost2"]}})))+ .await+ .unwrap();+ assert_eq!(+ r.total.value, 0,+ "query {query:?} over only unmapped fields"+ );+ }+}++/// The same zeroing hit every field-centric type through the shared+/// lowering; `most_fields` is the non-dis_max shape.+#[tokio::test]+async fn most_fields_multi_token_with_unmapped_field() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mm217_most").await;++ let r = idx+ .search(&req(json!({"multi_match": {+ "query": "merge zzzznotpresent",+ "fields": ["body", "ghost"],+ "type": "most_fields"}})))+ .await+ .unwrap();+ assert_eq!(r.total.value, 1, "most_fields shares the lowering (#217)");+}++/// `operator: and` binds PER FIELD (ES best_fields/most_fields build one+/// match query per field with the operator): a document matches only when+/// ONE field holds every token. The pre-fix FTS lowering ignored the+/// operator and OR'd the tokens, over-matching whenever every named field+/// was mapped.+#[tokio::test]+async fn operator_and_requires_every_token_in_one_field() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mm217_and").await;++ // No single field holds both "merge" and "zzzznotpresent" → 0 hits.+ let r = idx+ .search(&req(json!({"multi_match": {+ "query": "merge zzzznotpresent",+ "fields": ["body", "title"],+ "operator": "and"}})))+ .await+ .unwrap();+ assert_eq!(+ r.total.value, 0,+ "operator:and must not degrade to OR over tokens"+ );++ // `body` holds both tokens; the unmapped field must not change that —+ // and in particular must not flip the query from AND onto an OR path.+ let r = idx+ .search(&req(json!({"multi_match": {+ "query": "merge policy",+ "fields": ["body", "ghost"],+ "operator": "and"}})))+ .await+ .unwrap();+ assert_eq!(r.total.value, 1, "mapped subset satisfies per-field AND");+}++/// `cross_fields` + `operator: and` is term-centric — every token must+/// appear in at least ONE of the (mapped) fields, NOT all in the same one.+/// Here `bravo` lives only in `alpha` and `charlie` only in `beta`, so a+/// per-field AND would find nothing while the combined view matches — with+/// or without an unmapped field in the list.+#[tokio::test]+async fn cross_fields_and_matches_across_fields() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("mm217_cross", Schema::empty()).unwrap();+ let idx = engine.get_index("mm217_cross").unwrap();+ idx.index_document(+ Some("1".to_string()),+ json!({ "alpha": "delta bravo", "beta": "charlie echo" }),+ )+ .await+ .unwrap();+ idx.flush().await.unwrap();++ for fields in [json!(["alpha", "beta"]), json!(["alpha", "beta", "ghost"])] {+ let r = idx+ .search(&req(json!({"multi_match": {+ "query": "bravo charlie",+ "fields": fields,+ "type": "cross_fields",+ "operator": "and"}})))+ .await+ .unwrap();+ assert_eq!(+ r.total.value, 1,+ "combined-text AND across mapped fields ({fields})"+ );+ }+}