Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): make keyword match/multi_match mapping-aware in the memtable (#354)

xerj-org27c3511798e6
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index aeab2da17..238e983ee 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -14005,7 +14005,10 @@ impl Index {
// from `_source` (#204 — accepted means honoured).
let resolved_query = {
let schema = self.schema.read().await;
- let resolved = rewrite_query_aliases(&request.query, &schema.schema);
+ let resolved = rewrite_keyword_full_text_to_term(
+ &rewrite_query_aliases(&request.query, &schema.schema),
+ &schema.schema,
+ );
if let Some(field) = unsearchable_query_field(&resolved, &schema.schema) {
return Err(EngineError::Common(xerj_common::XerjError::invalid_query(
format!(
@@ -32014,6 +32017,100 @@ mod lexically_typeless_lowering_tests {
}
}
+/// #354: `match`/`multi_match` on a `keyword` field must take the query text
+/// WHOLE (keyword analyzer), not whitespace-tokenise + OR it. The flushed
+/// segment path already does this (per-field analyzer), but the memtable's
+/// schema-less `doc_matches_query`/`score_query_against_doc` tokenise, so the
+/// answer flipped at `_flush`. Rewrite keyword-targeted `Match`/`MultiMatch`
+/// clauses to whole-value `Term`s once here — where the schema is in scope — so
+/// BOTH paths agree. Analyzed-`text` fields are left untouched; a mixed
+/// `multi_match` splits into `Term`s for the keyword fields plus a `multi_match`
+/// over the remaining text fields. Same traversal coverage as
+/// `rewrite_query_aliases`, and applied after it so field names are canonical.
+fn rewrite_keyword_full_text_to_term(q: &QueryNode, schema: &Schema) -> QueryNode {
+ fn is_keyword(schema: &Schema, field: &str) -> bool {
+ schema
+ .fields
+ .iter()
+ .any(|f| f.name == field && matches!(f.field_type, FieldType::Keyword))
+ }
+ match q {
+ QueryNode::Match {
+ field,
+ query,
+ boost,
+ ..
+ } if is_keyword(schema, field) => QueryNode::Term {
+ field: field.clone(),
+ value: Value::String(query.clone()),
+ boost: *boost,
+ },
+ QueryNode::MultiMatch {
+ fields,
+ query,
+ boost,
+ ..
+ } if fields.iter().any(|f| is_keyword(schema, f)) => {
+ let (kw, text): (Vec<String>, Vec<String>) =
+ fields.iter().cloned().partition(|f| is_keyword(schema, f));
+ let mut should: Vec<QueryNode> = kw
+ .into_iter()
+ .map(|f| QueryNode::Term {
+ field: f,
+ value: Value::String(query.clone()),
+ boost: *boost,
+ })
+ .collect();
+ if !text.is_empty() {
+ // Preserve match_type/operator/etc. by cloning and swapping only
+ // the field list.
+ let mut text_mm = q.clone();
+ if let QueryNode::MultiMatch { fields: mmf, .. } = &mut text_mm {
+ *mmf = text;
+ }
+ should.push(text_mm);
+ }
+ if should.len() == 1 {
+ should.into_iter().next().unwrap()
+ } else {
+ QueryNode::Bool {
+ must: Vec::new(),
+ should,
+ must_not: Vec::new(),
+ filter: Vec::new(),
+ minimum_should_match: Some(MinShouldMatch::Fixed(1)),
+ }
+ }
+ }
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ minimum_should_match,
+ } => QueryNode::Bool {
+ must: must
+ .iter()
+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))
+ .collect(),
+ should: should
+ .iter()
+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))
+ .collect(),
+ must_not: must_not
+ .iter()
+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))
+ .collect(),
+ filter: filter
+ .iter()
+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))
+ .collect(),
+ minimum_should_match: minimum_should_match.clone(),
+ },
+ other => other.clone(),
+ }
+}
+
/// Rewrite a query by resolving any field aliases to their canonical field names.
///
/// Traverses the query tree, replacing alias field names with their target paths.
diff --git a/engine/crates/xerj-engine/tests/keyword_match_is_mapping_aware.rs b/engine/crates/xerj-engine/tests/keyword_match_is_mapping_aware.rs
new file mode 100644
index 000000000..8fc1541a7
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/keyword_match_is_mapping_aware.rs
@@ -0,0 +1,171 @@
+//! Issue #354: `match` / `multi_match` on a **keyword** field is mapping-aware
+//! in a flushed segment but NOT in the memtable, so a multi-token query changes
+//! its answer at `_flush`. No arrays are involved (independent of #332).
+//!
+//! `doc_matches_query` / `score_query_against_doc` (xerj-engine/src/index.rs)
+//! evaluate the `Match` / `MultiMatch` arms by whitespace-tokenising the query
+//! and OR-ing the tokens against the stored `_source`, never consulting the
+//! mapping — so a `keyword` field is treated like an analyzed `text` field. The
+//! segment side DOES consult the mapping (keyword analyzer), so the answer
+//! diverges at flush.
+//!
+//! One doc, `tags` mapped `keyword`, scalar value `"red"`:
+//! ```text
+//! BEFORE flush | multi_match 'red blue' -> 1 hit (WRONG: tokenised + OR-ed)
+//! AFTER flush | multi_match 'red blue' -> 0 hits (correct: "red" != "red blue")
+//! ```
+//! Elasticsearch takes the query text whole under the keyword analyzer, so
+//! `"red blue"` is one term, does not equal `"red"`, and the doc matches in
+//! NEITHER phase. This test asserts the correct (0-hit) answer in both phases.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is here.
+
+use std::collections::BTreeSet;
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn req(q: Value) -> xerj_query::ast::SearchRequest {
+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")
+}
+
+async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {
+ idx.search(&req(q.clone()))
+ .await
+ .unwrap()
+ .hits
+ .iter()
+ .map(|h| h.id.clone())
+ .collect()
+}
+
+fn expect(ids: &[&str]) -> BTreeSet<String> {
+ ids.iter().map(|s| s.to_string()).collect()
+}
+
+/// A multi-token `match`/`multi_match` over a scalar `keyword` field must match
+/// the value WHOLE (keyword analyzer) — so `"red blue"` never matches `"red"` —
+/// identically before and after `flush()`. Fixed by rewriting keyword-targeted
+/// `Match`/`MultiMatch` clauses to whole-value `Term`s at the search entry
+/// (`rewrite_keyword_full_text_to_term`), so the memtable agrees with the
+/// segment. Proven fail-before on main (8831d85c): PRE-flush `multi_match
+/// 'red blue'` returned {"2"} instead of {}.
+#[tokio::test]
+async fn keyword_match_takes_the_query_whole_in_both_phases() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("tags", FieldType::Keyword));
+ engine.create_index("kw", schema).unwrap();
+ let idx = engine.get_index("kw").unwrap();
+ idx.index_document(Some("2".into()), json!({ "tags": "red" }))
+ .await
+ .unwrap();
+
+ // (query, expected-ids, label). Same hit set required PRE- and POST-flush.
+ let cases: &[(Value, &[&str], &str)] = &[
+ (
+ json!({ "multi_match": { "query": "red blue", "fields": ["tags"] } }),
+ &[],
+ "multi_match 'red blue' on scalar keyword",
+ ),
+ (
+ json!({ "match": { "tags": "red blue" } }),
+ &[],
+ "match 'red blue' on scalar keyword",
+ ),
+ // Control: the whole value DOES match, in both phases.
+ (
+ json!({ "term": { "tags": "red" } }),
+ &["2"],
+ "term tags=red (control)",
+ ),
+ ];
+
+ for (q, exp, label) in cases {
+ assert_eq!(
+ ids(&idx, q).await,
+ expect(exp),
+ "{label}: PRE-flush (memtable) hit set wrong for {q} — a keyword field \
+ must take the query text whole, not whitespace-tokenise + OR it (#354)"
+ );
+ }
+ idx.flush().await.unwrap();
+ for (q, exp, label) in cases {
+ assert_eq!(
+ ids(&idx, q).await,
+ expect(exp),
+ "{label}: POST-flush (segment) hit set wrong for {q}"
+ );
+ }
+}
+
+/// A `multi_match` spanning BOTH a keyword and a text field must split: the
+/// keyword field takes the query WHOLE (so `"red blue"` misses `"red"`), while
+/// the text field stays analyzed (so a shared token still matches) — identically
+/// before and after flush.
+#[tokio::test]
+async fn mixed_multi_match_splits_keyword_and_text() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("tags", FieldType::Keyword));
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ engine.create_index("mm", schema).unwrap();
+ let idx = engine.get_index("mm").unwrap();
+ // "1": keyword "red" (whole), text "blue sky" (analyzed → tokens blue, sky).
+ idx.index_document(
+ Some("1".into()),
+ json!({ "tags": "red", "body": "blue sky" }),
+ )
+ .await
+ .unwrap();
+ // "2": keyword "green", text "hello world" — matches neither half of "red blue".
+ idx.index_document(
+ Some("2".into()),
+ json!({ "tags": "green", "body": "hello world" }),
+ )
+ .await
+ .unwrap();
+
+ // multi_match "red blue" over [tags(keyword), body(text)]:
+ // tags whole "red blue" != "red" -> no keyword match on "1"
+ // body analyzed has token "blue" -> text match on "1"
+ // So only "1" matches (via the text half); "2" matches neither.
+ let q = json!({ "multi_match": { "query": "red blue", "fields": ["tags", "body"] } });
+ let cases: &[(Value, &[&str], &str)] = &[(q, &["1"], "mixed multi_match 'red blue'")];
+ for (q, exp, label) in cases {
+ assert_eq!(
+ ids(&idx, q).await,
+ expect(exp),
+ "{label}: PRE-flush hit set wrong for {q} — keyword half must be whole-value, \
+ text half analyzed (#354)"
+ );
+ }
+ idx.flush().await.unwrap();
+ for (q, exp, label) in cases {
+ assert_eq!(
+ ids(&idx, q).await,
+ expect(exp),
+ "{label}: POST-flush hit set wrong for {q}"
+ );
+ }
+}