AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): make keyword match/multi_match mapping-aware inside Knn/SemanticSearch/function_score filters (#574)
xerj-org6fa09a131fc3
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex fa0e30e86..060c605f6 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -32413,7 +32413,21 @@ fn rewrite_keyword_full_text_to_term(q: &QueryNode, schema: &Schema) -> QueryNodmax_boost,} => QueryNode::FunctionScore {query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),- functions: functions.clone(),+ // Each function may carry its own `filter` sub-query targeting a+ // keyword field; recurse into it too (#574) — otherwise a keyword+ // `match`/`multi_match` inside a `function_score` function filter+ // keeps flipping its hit set at flush (#354).+ functions: functions+ .iter()+ .map(|f| {+ let mut nf = f.clone();+ nf.filter = f+ .filter+ .as_ref()+ .map(|q| rewrite_keyword_full_text_to_term(q, schema));+ nf+ })+ .collect(),score_mode: *score_mode,boost_mode: *boost_mode,max_boost: *max_boost,@@ -32464,10 +32478,178 @@ fn rewrite_keyword_full_text_to_term(q: &QueryNode, schema: &Schema) -> QueryNodname: name.clone(),query: Box::new(rewrite_keyword_full_text_to_term(query, schema)),},+ // `Knn`/`SemanticSearch` carry an optional `filter` sub-query that gates+ // which docs the vector search runs against; a keyword `match`/+ // `multi_match` in that filter must be made mapping-aware too, or it+ // flips its hit set at flush (#354/#574). Only the filter is a full-text+ // sub-query — the vector/text search itself is not rewritten.+ QueryNode::Knn {+ field,+ vector,+ k,+ num_candidates,+ filter,+ boost,+ similarity,+ } => QueryNode::Knn {+ field: field.clone(),+ vector: vector.clone(),+ k: *k,+ num_candidates: *num_candidates,+ filter: filter+ .as_ref()+ .map(|f| Box::new(rewrite_keyword_full_text_to_term(f, schema))),+ boost: *boost,+ similarity: *similarity,+ },+ QueryNode::SemanticSearch {+ field,+ text,+ k,+ filter,+ boost,+ } => QueryNode::SemanticSearch {+ field: field.clone(),+ text: text.clone(),+ k: *k,+ filter: filter+ .as_ref()+ .map(|f| Box::new(rewrite_keyword_full_text_to_term(f, schema))),+ boost: *boost,+ },+ // Leaf nodes (Term/Terms/Range/Exists/...) and positional (span) nodes+ // carry no full-text sub-query to rewrite and fall through unchanged.other => other.clone(),}}+#[cfg(test)]+mod keyword_rewrite_filter_tests {+ use super::*;++ fn kw_schema() -> Schema {+ let mut s = Schema::empty();+ s.fields.push(FieldConfig::new("kw", FieldType::Keyword));+ s+ }++ // A multi-token keyword `match`: "a b" must be rewritten to a whole-value+ // `Term`, or it whitespace-tokenises + OR-s and flips its hit set at flush.+ fn kw_match() -> QueryNode {+ QueryNode::Match {+ field: "kw".to_string(),+ query: "a b".to_string(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ }+ }++ fn assert_is_kw_term(q: &QueryNode) {+ match q {+ QueryNode::Term { field, value, .. } => {+ assert_eq!(field, "kw");+ assert_eq!(value, &Value::String("a b".to_string()));+ }+ other => panic!("expected keyword Match rewritten to whole-value Term, got {other:?}"),+ }+ }++ /// #574: a keyword `match`/`multi_match` inside a `Knn`/`SemanticSearch`/+ /// `function_score` function **filter** must be made mapping-aware (rewritten+ /// to a whole-value `Term`) like the top-level and compound-wrapper cases+ /// (#571/#573) — otherwise it flips its hit set at flush (#354). These nodes+ /// previously fell through `other => other.clone()` (Knn/SemanticSearch) or+ /// cloned `functions` verbatim (function_score).+ #[test]+ fn keyword_match_in_filter_carrying_nodes_is_rewritten() {+ let schema = kw_schema();++ // Knn.filter+ let knn = QueryNode::Knn {+ field: "vec".to_string(),+ vector: vec![0.1, 0.2],+ k: 5,+ num_candidates: None,+ filter: Some(Box::new(kw_match())),+ boost: None,+ similarity: None,+ };+ match rewrite_keyword_full_text_to_term(&knn, &schema) {+ QueryNode::Knn {+ filter: Some(f), ..+ } => assert_is_kw_term(&f),+ other => panic!("Knn shape changed: {other:?}"),+ }++ // SemanticSearch.filter+ let sem = QueryNode::SemanticSearch {+ field: "vec".to_string(),+ text: "hello".to_string(),+ k: 5,+ filter: Some(Box::new(kw_match())),+ boost: None,+ };+ match rewrite_keyword_full_text_to_term(&sem, &schema) {+ QueryNode::SemanticSearch {+ filter: Some(f), ..+ } => assert_is_kw_term(&f),+ other => panic!("SemanticSearch shape changed: {other:?}"),+ }++ // function_score functions[].filter+ let fs = QueryNode::FunctionScore {+ query: Box::new(QueryNode::Term {+ field: "other".to_string(),+ value: Value::String("x".to_string()),+ boost: None,+ }),+ functions: vec![ScoreFunction {+ filter: Some(kw_match()),+ ..Default::default()+ }],+ score_mode: Default::default(),+ boost_mode: Default::default(),+ max_boost: None,+ };+ match rewrite_keyword_full_text_to_term(&fs, &schema) {+ QueryNode::FunctionScore { functions, .. } => {+ assert_is_kw_term(functions[0].filter.as_ref().expect("filter preserved"))+ }+ other => panic!("FunctionScore shape changed: {other:?}"),+ }++ // A NON-keyword match in a filter is left as a Match (only keyword fields+ // are rewritten; text fields stay analyzed).+ let knn_text = QueryNode::Knn {+ field: "vec".to_string(),+ vector: vec![0.1],+ k: 1,+ num_candidates: None,+ filter: Some(Box::new(QueryNode::Match {+ field: "body".to_string(),+ query: "a b".to_string(),+ operator: Default::default(),+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ })),+ boost: None,+ similarity: None,+ };+ match rewrite_keyword_full_text_to_term(&knn_text, &schema) {+ QueryNode::Knn {+ filter: Some(f), ..+ } => assert!(+ matches!(*f, QueryNode::Match { .. }),+ "a non-keyword field must stay a Match, got {f:?}"+ ),+ other => panic!("Knn shape changed: {other:?}"),+ }+ }+}+/// Rewrite a query by resolving any field aliases to their canonical field names.////// Traverses the query tree, replacing alias field names with their target paths.