Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): make keyword match/multi_match mapping-aware inside wrapper queries

xerj-org2d9f35eed7b9
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index a6bb013ae..38ffe3e89 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -23,7 +23,7 @@ use xerj_fts::search::{
use xerj_query::ast::{
BoostMode, FieldValueFactor, Fuzziness, HighlightRequest, MinShouldMatch, Modifier, QueryNode,
RandomScore, RescoreQuery, SavingsMode, ScoreFunction, ScoreMode, SearchRequest, SourceFilter,
- TrackTotalHits,
+ TrackTotalHits, WeightedQuery,
};
use xerj_query::executor::{
strip_internal_passage_metadata, Hit, PassageMatch, PayloadSavings, SavingsMethod,
@@ -32128,6 +32128,78 @@ fn rewrite_keyword_full_text_to_term(q: &QueryNode, schema: &Schema) -> QueryNod
.collect(),
minimum_should_match: minimum_should_match.clone(),
},
+ // Wrapper/compound nodes carry sub-queries that may target keyword
+ // fields; recurse into them so a keyword `match`/`multi_match` nested
+ // inside them is made mapping-aware too (otherwise it keeps flipping at
+ // flush). Leaf and positional (span) nodes have no full-text sub-query
+ // to rewrite and fall through unchanged.
+ QueryNode::Constant { score, query } => QueryNode::Constant {
+ score: *score,
+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),
+ },
+ QueryNode::Boosted { boost, query } => QueryNode::Boosted {
+ boost: *boost,
+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),
+ },
+ QueryNode::FunctionScore {
+ query,
+ functions,
+ score_mode,
+ boost_mode,
+ max_boost,
+ } => QueryNode::FunctionScore {
+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),
+ functions: functions.clone(),
+ score_mode: *score_mode,
+ boost_mode: *boost_mode,
+ max_boost: *max_boost,
+ },
+ QueryNode::Boosting {
+ positive,
+ negative,
+ negative_boost,
+ } => QueryNode::Boosting {
+ positive: Box::new(rewrite_keyword_full_text_to_term(positive, schema)),
+ negative: Box::new(rewrite_keyword_full_text_to_term(negative, schema)),
+ negative_boost: *negative_boost,
+ },
+ QueryNode::DisMax {
+ queries,
+ tie_breaker,
+ } => QueryNode::DisMax {
+ queries: queries
+ .iter()
+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))
+ .collect(),
+ tie_breaker: *tie_breaker,
+ },
+ QueryNode::Nested {
+ path,
+ query,
+ score_mode,
+ } => QueryNode::Nested {
+ path: path.clone(),
+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),
+ score_mode: score_mode.clone(),
+ },
+ QueryNode::Hybrid { queries, fusion } => QueryNode::Hybrid {
+ queries: queries
+ .iter()
+ .map(|wq| WeightedQuery {
+ query: rewrite_keyword_full_text_to_term(&wq.query, schema),
+ weight: wq.weight,
+ })
+ .collect(),
+ fusion: fusion.clone(),
+ },
+ QueryNode::Pinned { ids, organic } => QueryNode::Pinned {
+ ids: ids.clone(),
+ organic: Box::new(rewrite_keyword_full_text_to_term(organic, schema)),
+ },
+ QueryNode::Named { name, query } => QueryNode::Named {
+ name: name.clone(),
+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),
+ },
other => other.clone(),
}
}
diff --git a/engine/crates/xerj-engine/tests/keyword_match_mapping_aware_in_wrappers.rs b/engine/crates/xerj-engine/tests/keyword_match_mapping_aware_in_wrappers.rs
new file mode 100644
index 000000000..632e49bdc
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/keyword_match_mapping_aware_in_wrappers.rs
@@ -0,0 +1,130 @@
+//! Issue #354 (follow-up): a keyword `match`/`multi_match` nested inside a
+//! wrapper/compound query — `constant_score`, `function_score`, `boosting`,
+//! `dis_max`, `nested`, `hybrid`, `pinned`, `named` — must be mapping-aware in
+//! BOTH the memtable and a flushed segment, exactly like a top-level clause.
+//!
+//! PR #571 rewrote top-level keyword `Match`/`MultiMatch` to whole-value `Term`s
+//! but recursed only into `Bool`, so a keyword match wrapped in e.g.
+//! `constant_score` fell through unchanged and kept flipping its hit set at
+//! `_flush` (the #354 defect, live for wrappers). This extends the rewrite's
+//! recursion to the sub-query-carrying wrappers.
+//!
+//! One doc, `tags` mapped `keyword`, scalar value `"red"`. A wrapped
+//! `match {tags: "red blue"}` must take the value WHOLE — `"red blue" != "red"`
+//! — so the doc matches in NEITHER phase.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is here.
+
+use std::collections::BTreeSet;
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn req(q: Value) -> xerj_query::ast::SearchRequest {
+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")
+}
+
+async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {
+ idx.search(&req(q.clone()))
+ .await
+ .unwrap()
+ .hits
+ .iter()
+ .map(|h| h.id.clone())
+ .collect()
+}
+
+fn expect(ids: &[&str]) -> BTreeSet<String> {
+ ids.iter().map(|s| s.to_string()).collect()
+}
+
+/// A keyword `match`/`multi_match` nested inside a wrapper query takes the
+/// query text WHOLE — so `"red blue"` never matches `"red"` — identically
+/// before and after `flush()`.
+#[tokio::test]
+async fn keyword_match_in_wrappers_takes_the_query_whole_in_both_phases() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("tags", FieldType::Keyword));
+ engine.create_index("kw", schema).unwrap();
+ let idx = engine.get_index("kw").unwrap();
+ idx.index_document(Some("2".into()), json!({ "tags": "red" }))
+ .await
+ .unwrap();
+
+ // Each wrapper embeds `match {tags: "red blue"}`; the keyword field takes
+ // the query whole, so none match in either phase. (query, expected, label)
+ let miss: &[(Value, &[&str], &str)] = &[
+ (
+ json!({ "constant_score": { "filter": { "match": { "tags": "red blue" } } } }),
+ &[],
+ "constant_score > match",
+ ),
+ (
+ json!({ "function_score": { "query": { "match": { "tags": "red blue" } } } }),
+ &[],
+ "function_score > match",
+ ),
+ (
+ json!({ "boosting": {
+ "positive": { "match": { "tags": "red blue" } },
+ "negative": { "match": { "tags": "zzz" } },
+ "negative_boost": 0.5
+ } }),
+ &[],
+ "boosting > positive match",
+ ),
+ (
+ json!({ "dis_max": { "queries": [ { "match": { "tags": "red blue" } } ] } }),
+ &[],
+ "dis_max > match",
+ ),
+ (
+ json!({ "dis_max": { "queries": [
+ { "multi_match": { "query": "red blue", "fields": ["tags^2"] } }
+ ] } }),
+ &[],
+ "dis_max > boosted multi_match",
+ ),
+ ];
+
+ // Positive control: the whole value DOES match, wrapped, in both phases.
+ let hit: &[(Value, &[&str], &str)] = &[(
+ json!({ "constant_score": { "filter": { "match": { "tags": "red" } } } }),
+ &["2"],
+ "constant_score > match (whole value, control)",
+ )];
+
+ let all: Vec<(Value, &[&str], &str)> = miss.iter().chain(hit.iter()).cloned().collect();
+
+ for (q, exp, label) in &all {
+ assert_eq!(
+ ids(&idx, q).await,
+ expect(exp),
+ "{label}: PRE-flush (memtable) hit set wrong for {q} — a keyword field \
+ wrapped in a compound query must take the query text whole (#354)"
+ );
+ }
+ idx.flush().await.unwrap();
+ for (q, exp, label) in &all {
+ assert_eq!(
+ ids(&idx, q).await,
+ expect(exp),
+ "{label}: POST-flush (segment) hit set wrong for {q}"
+ );
+ }
+}