AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): make keyword match/multi_match mapping-aware inside wrapper queries
xerj-org2d9f35eed7b9
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex a6bb013ae..38ffe3e89 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -23,7 +23,7 @@ use xerj_fts::search::{use xerj_query::ast::{BoostMode, FieldValueFactor, Fuzziness, HighlightRequest, MinShouldMatch, Modifier, QueryNode,RandomScore, RescoreQuery, SavingsMode, ScoreFunction, ScoreMode, SearchRequest, SourceFilter,- TrackTotalHits,+ TrackTotalHits, WeightedQuery,};use xerj_query::executor::{strip_internal_passage_metadata, Hit, PassageMatch, PayloadSavings, SavingsMethod,@@ -32128,6 +32128,78 @@ fn rewrite_keyword_full_text_to_term(q: &QueryNode, schema: &Schema) -> QueryNod.collect(),minimum_should_match: minimum_should_match.clone(),},+ // Wrapper/compound nodes carry sub-queries that may target keyword+ // fields; recurse into them so a keyword `match`/`multi_match` nested+ // inside them is made mapping-aware too (otherwise it keeps flipping at+ // flush). Leaf and positional (span) nodes have no full-text sub-query+ // to rewrite and fall through unchanged.+ QueryNode::Constant { score, query } => QueryNode::Constant {+ score: *score,+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),+ },+ QueryNode::Boosted { boost, query } => QueryNode::Boosted {+ boost: *boost,+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),+ },+ QueryNode::FunctionScore {+ query,+ functions,+ score_mode,+ boost_mode,+ max_boost,+ } => QueryNode::FunctionScore {+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),+ functions: functions.clone(),+ score_mode: *score_mode,+ boost_mode: *boost_mode,+ max_boost: *max_boost,+ },+ QueryNode::Boosting {+ positive,+ negative,+ negative_boost,+ } => QueryNode::Boosting {+ positive: Box::new(rewrite_keyword_full_text_to_term(positive, schema)),+ negative: Box::new(rewrite_keyword_full_text_to_term(negative, schema)),+ negative_boost: *negative_boost,+ },+ QueryNode::DisMax {+ queries,+ tie_breaker,+ } => QueryNode::DisMax {+ queries: queries+ .iter()+ .map(|c| rewrite_keyword_full_text_to_term(c, schema))+ .collect(),+ tie_breaker: *tie_breaker,+ },+ QueryNode::Nested {+ path,+ query,+ score_mode,+ } => QueryNode::Nested {+ path: path.clone(),+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),+ score_mode: score_mode.clone(),+ },+ QueryNode::Hybrid { queries, fusion } => QueryNode::Hybrid {+ queries: queries+ .iter()+ .map(|wq| WeightedQuery {+ query: rewrite_keyword_full_text_to_term(&wq.query, schema),+ weight: wq.weight,+ })+ .collect(),+ fusion: fusion.clone(),+ },+ QueryNode::Pinned { ids, organic } => QueryNode::Pinned {+ ids: ids.clone(),+ organic: Box::new(rewrite_keyword_full_text_to_term(organic, schema)),+ },+ QueryNode::Named { name, query } => QueryNode::Named {+ name: name.clone(),+ query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),+ },other => other.clone(),}}diff --git a/engine/crates/xerj-engine/tests/keyword_match_mapping_aware_in_wrappers.rs b/engine/crates/xerj-engine/tests/keyword_match_mapping_aware_in_wrappers.rsnew file mode 100644index 000000000..632e49bdc--- /dev/null+++ b/engine/crates/xerj-engine/tests/keyword_match_mapping_aware_in_wrappers.rs@@ -0,0 +1,130 @@+//! Issue #354 (follow-up): a keyword `match`/`multi_match` nested inside a+//! wrapper/compound query — `constant_score`, `function_score`, `boosting`,+//! `dis_max`, `nested`, `hybrid`, `pinned`, `named` — must be mapping-aware in+//! BOTH the memtable and a flushed segment, exactly like a top-level clause.+//!+//! PR #571 rewrote top-level keyword `Match`/`MultiMatch` to whole-value `Term`s+//! but recursed only into `Bool`, so a keyword match wrapped in e.g.+//! `constant_score` fell through unchanged and kept flipping its hit set at+//! `_flush` (the #354 defect, live for wrappers). This extends the rewrite's+//! recursion to the sub-query-carrying wrappers.+//!+//! One doc, `tags` mapped `keyword`, scalar value `"red"`. A wrapped+//! `match {tags: "red blue"}` must take the value WHOLE — `"red blue" != "red"`+//! — so the doc matches in NEITHER phase.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is here.++use std::collections::BTreeSet;++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn req(q: Value) -> xerj_query::ast::SearchRequest {+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")+}++async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {+ idx.search(&req(q.clone()))+ .await+ .unwrap()+ .hits+ .iter()+ .map(|h| h.id.clone())+ .collect()+}++fn expect(ids: &[&str]) -> BTreeSet<String> {+ ids.iter().map(|s| s.to_string()).collect()+}++/// A keyword `match`/`multi_match` nested inside a wrapper query takes the+/// query text WHOLE — so `"red blue"` never matches `"red"` — identically+/// before and after `flush()`.+#[tokio::test]+async fn keyword_match_in_wrappers_takes_the_query_whole_in_both_phases() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("tags", FieldType::Keyword));+ engine.create_index("kw", schema).unwrap();+ let idx = engine.get_index("kw").unwrap();+ idx.index_document(Some("2".into()), json!({ "tags": "red" }))+ .await+ .unwrap();++ // Each wrapper embeds `match {tags: "red blue"}`; the keyword field takes+ // the query whole, so none match in either phase. (query, expected, label)+ let miss: &[(Value, &[&str], &str)] = &[+ (+ json!({ "constant_score": { "filter": { "match": { "tags": "red blue" } } } }),+ &[],+ "constant_score > match",+ ),+ (+ json!({ "function_score": { "query": { "match": { "tags": "red blue" } } } }),+ &[],+ "function_score > match",+ ),+ (+ json!({ "boosting": {+ "positive": { "match": { "tags": "red blue" } },+ "negative": { "match": { "tags": "zzz" } },+ "negative_boost": 0.5+ } }),+ &[],+ "boosting > positive match",+ ),+ (+ json!({ "dis_max": { "queries": [ { "match": { "tags": "red blue" } } ] } }),+ &[],+ "dis_max > match",+ ),+ (+ json!({ "dis_max": { "queries": [+ { "multi_match": { "query": "red blue", "fields": ["tags^2"] } }+ ] } }),+ &[],+ "dis_max > boosted multi_match",+ ),+ ];++ // Positive control: the whole value DOES match, wrapped, in both phases.+ let hit: &[(Value, &[&str], &str)] = &[(+ json!({ "constant_score": { "filter": { "match": { "tags": "red" } } } }),+ &["2"],+ "constant_score > match (whole value, control)",+ )];++ let all: Vec<(Value, &[&str], &str)> = miss.iter().chain(hit.iter()).cloned().collect();++ for (q, exp, label) in &all {+ assert_eq!(+ ids(&idx, q).await,+ expect(exp),+ "{label}: PRE-flush (memtable) hit set wrong for {q} — a keyword field \+ wrapped in a compound query must take the query text whole (#354)"+ );+ }+ idx.flush().await.unwrap();+ for (q, exp, label) in &all {+ assert_eq!(+ ids(&idx, q).await,+ expect(exp),+ "{label}: POST-flush (segment) hit set wrong for {q}"+ );+ }+}