AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): make keyword match/multi_match mapping-aware in the memtable (#354)
xerj-org27c3511798e6
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex aeab2da17..238e983ee 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -14005,7 +14005,10 @@ impl Index {// from `_source` (#204 — accepted means honoured).let resolved_query = {let schema = self.schema.read().await;- let resolved = rewrite_query_aliases(&request.query, &schema.schema);+ let resolved = rewrite_keyword_full_text_to_term(+ &rewrite_query_aliases(&request.query, &schema.schema),+ &schema.schema,+ );if let Some(field) = unsearchable_query_field(&resolved, &schema.schema) {return Err(EngineError::Common(xerj_common::XerjError::invalid_query(format!(@@ -32014,6 +32017,100 @@ mod lexically_typeless_lowering_tests {}}+/// #354: `match`/`multi_match` on a `keyword` field must take the query text+/// WHOLE (keyword analyzer), not whitespace-tokenise + OR it. The flushed+/// segment path already does this (per-field analyzer), but the memtable's+/// schema-less `doc_matches_query`/`score_query_against_doc` tokenise, so the+/// answer flipped at `_flush`. Rewrite keyword-targeted `Match`/`MultiMatch`+/// clauses to whole-value `Term`s once here — where the schema is in scope — so+/// BOTH paths agree. Analyzed-`text` fields are left untouched; a mixed+/// `multi_match` splits into `Term`s for the keyword fields plus a `multi_match`+/// over the remaining text fields. Same traversal coverage as+/// `rewrite_query_aliases`, and applied after it so field names are canonical.+fn rewrite_keyword_full_text_to_term(q: &QueryNode, schema: &Schema) -> QueryNode {+ fn is_keyword(schema: &Schema, field: &str) -> bool {+ schema+ .fields+ .iter()+ .any(|f| f.name == field && matches!(f.field_type, FieldType::Keyword))+ }+ match q {+ QueryNode::Match {+ field,+ query,+ boost,+ ..+ } if is_keyword(schema, field) => QueryNode::Term {+ field: field.clone(),+ value: Value::String(query.clone()),+ boost: *boost,+ },+ QueryNode::MultiMatch {+ fields,+ query,+ boost,+ ..+ } if fields.iter().any(|f| is_keyword(schema, f)) => {+ let (kw, text): (Vec<String>, Vec<String>) =+ fields.iter().cloned().partition(|f| is_keyword(schema, f));+ let mut should: Vec<QueryNode> = kw+ .into_iter()+ .map(|f| QueryNode::Term {+ field: f,+ value: Value::String(query.clone()),+ boost: *boost,+ })+ .collect();+ if !text.is_empty() {+ // Preserve match_type/operator/etc. by cloning and swapping only+ // the field list.+ let mut text_mm = q.clone();+ if let QueryNode::MultiMatch { fields: mmf, .. } = &mut text_mm {+ *mmf = text;+ }+ should.push(text_mm);+ }+ if should.len() == 1 {+ should.into_iter().next().unwrap()+ } else {+ QueryNode::Bool {+ must: Vec::new(),+ should,+ must_not: Vec::new(),+ filter: Vec::new(),+ minimum_should_match: Some(MinShouldMatch::Fixed(1)),+ }+ }+ }+ QueryNode::Bool {+ must,+ should,+ must_not,+ filter,+ minimum_should_match,+ } => QueryNode::Bool {+ must: must+ .iter()+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))+ .collect(),+ should: should+ .iter()+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))+ .collect(),+ must_not: must_not+ .iter()+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))+ .collect(),+ filter: filter+ .iter()+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))+ .collect(),+ minimum_should_match: minimum_should_match.clone(),+ },+ other => other.clone(),+ }+}+/// Rewrite a query by resolving any field aliases to their canonical field names.////// Traverses the query tree, replacing alias field names with their target paths.diff --git a/engine/crates/xerj-engine/tests/keyword_match_is_mapping_aware.rs b/engine/crates/xerj-engine/tests/keyword_match_is_mapping_aware.rsnew file mode 100644index 000000000..8fc1541a7--- /dev/null+++ b/engine/crates/xerj-engine/tests/keyword_match_is_mapping_aware.rs@@ -0,0 +1,171 @@+//! Issue #354: `match` / `multi_match` on a **keyword** field is mapping-aware+//! in a flushed segment but NOT in the memtable, so a multi-token query changes+//! its answer at `_flush`. No arrays are involved (independent of #332).+//!+//! `doc_matches_query` / `score_query_against_doc` (xerj-engine/src/index.rs)+//! evaluate the `Match` / `MultiMatch` arms by whitespace-tokenising the query+//! and OR-ing the tokens against the stored `_source`, never consulting the+//! mapping — so a `keyword` field is treated like an analyzed `text` field. The+//! segment side DOES consult the mapping (keyword analyzer), so the answer+//! diverges at flush.+//!+//! One doc, `tags` mapped `keyword`, scalar value `"red"`:+//! ```text+//! BEFORE flush | multi_match 'red blue' -> 1 hit (WRONG: tokenised + OR-ed)+//! AFTER flush | multi_match 'red blue' -> 0 hits (correct: "red" != "red blue")+//! ```+//! Elasticsearch takes the query text whole under the keyword analyzer, so+//! `"red blue"` is one term, does not equal `"red"`, and the doc matches in+//! NEITHER phase. This test asserts the correct (0-hit) answer in both phases.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is here.++use std::collections::BTreeSet;++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn req(q: Value) -> xerj_query::ast::SearchRequest {+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")+}++async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {+ idx.search(&req(q.clone()))+ .await+ .unwrap()+ .hits+ .iter()+ .map(|h| h.id.clone())+ .collect()+}++fn expect(ids: &[&str]) -> BTreeSet<String> {+ ids.iter().map(|s| s.to_string()).collect()+}++/// A multi-token `match`/`multi_match` over a scalar `keyword` field must match+/// the value WHOLE (keyword analyzer) — so `"red blue"` never matches `"red"` —+/// identically before and after `flush()`. Fixed by rewriting keyword-targeted+/// `Match`/`MultiMatch` clauses to whole-value `Term`s at the search entry+/// (`rewrite_keyword_full_text_to_term`), so the memtable agrees with the+/// segment. Proven fail-before on main (8831d85c): PRE-flush `multi_match+/// 'red blue'` returned {"2"} instead of {}.+#[tokio::test]+async fn keyword_match_takes_the_query_whole_in_both_phases() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("tags", FieldType::Keyword));+ engine.create_index("kw", schema).unwrap();+ let idx = engine.get_index("kw").unwrap();+ idx.index_document(Some("2".into()), json!({ "tags": "red" }))+ .await+ .unwrap();++ // (query, expected-ids, label). Same hit set required PRE- and POST-flush.+ let cases: &[(Value, &[&str], &str)] = &[+ (+ json!({ "multi_match": { "query": "red blue", "fields": ["tags"] } }),+ &[],+ "multi_match 'red blue' on scalar keyword",+ ),+ (+ json!({ "match": { "tags": "red blue" } }),+ &[],+ "match 'red blue' on scalar keyword",+ ),+ // Control: the whole value DOES match, in both phases.+ (+ json!({ "term": { "tags": "red" } }),+ &["2"],+ "term tags=red (control)",+ ),+ ];++ for (q, exp, label) in cases {+ assert_eq!(+ ids(&idx, q).await,+ expect(exp),+ "{label}: PRE-flush (memtable) hit set wrong for {q} — a keyword field \+ must take the query text whole, not whitespace-tokenise + OR it (#354)"+ );+ }+ idx.flush().await.unwrap();+ for (q, exp, label) in cases {+ assert_eq!(+ ids(&idx, q).await,+ expect(exp),+ "{label}: POST-flush (segment) hit set wrong for {q}"+ );+ }+}++/// A `multi_match` spanning BOTH a keyword and a text field must split: the+/// keyword field takes the query WHOLE (so `"red blue"` misses `"red"`), while+/// the text field stays analyzed (so a shared token still matches) — identically+/// before and after flush.+#[tokio::test]+async fn mixed_multi_match_splits_keyword_and_text() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("tags", FieldType::Keyword));+ schema+ .fields+ .push(FieldConfig::new("body", FieldType::Text));+ engine.create_index("mm", schema).unwrap();+ let idx = engine.get_index("mm").unwrap();+ // "1": keyword "red" (whole), text "blue sky" (analyzed → tokens blue, sky).+ idx.index_document(+ Some("1".into()),+ json!({ "tags": "red", "body": "blue sky" }),+ )+ .await+ .unwrap();+ // "2": keyword "green", text "hello world" — matches neither half of "red blue".+ idx.index_document(+ Some("2".into()),+ json!({ "tags": "green", "body": "hello world" }),+ )+ .await+ .unwrap();++ // multi_match "red blue" over [tags(keyword), body(text)]:+ // tags whole "red blue" != "red" -> no keyword match on "1"+ // body analyzed has token "blue" -> text match on "1"+ // So only "1" matches (via the text half); "2" matches neither.+ let q = json!({ "multi_match": { "query": "red blue", "fields": ["tags", "body"] } });+ let cases: &[(Value, &[&str], &str)] = &[(q, &["1"], "mixed multi_match 'red blue'")];+ for (q, exp, label) in cases {+ assert_eq!(+ ids(&idx, q).await,+ expect(exp),+ "{label}: PRE-flush hit set wrong for {q} — keyword half must be whole-value, \+ text half analyzed (#354)"+ );+ }+ idx.flush().await.unwrap();+ for (q, exp, label) in cases {+ assert_eq!(+ ids(&idx, q).await,+ expect(exp),+ "{label}: POST-flush hit set wrong for {q}"+ );+ }+}