AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): preserve bool keyword-term scoring
buger4683fe2acd9b
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 2187c51d7..f5938cc63 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -14255,6 +14255,7 @@ impl Index {query,&text_fields,&exact_fields,+ &kw_fields,mem_doc_count,).map(Arc::new)@@ -14878,7 +14879,9 @@ impl Index {// and materialises only the top prefix, so the F1 bounded-scan / count// overwrite must NOT touch FTS queries — it only applies to the// non-FTS stored-doc scan (match_all / term-on-keyword / range).- let query_needs_fts: bool = query_node_to_fts(query, &text_fields, &exact_fields).is_some();+ let query_needs_fts: bool =+ query_node_to_fts_with_keyword_fields(query, &text_fields, &exact_fields, &kw_fields)+ .is_some();// ── Precomputed segment agg fast path (M2 G2) ─────────────────────//@@ -15122,7 +15125,12 @@ impl Index {// entirely — that call eagerly reads .fst/.meta/.post/.norms// for every text field, which costs ~50 MB per segment and is// wasted work when the query is a stored-doc scan.- let fts_query_probe = query_node_to_fts(query, &text_fields, &exact_fields);+ let fts_query_probe = query_node_to_fts_with_keyword_fields(+ query,+ &text_fields,+ &exact_fields,+ &kw_fields,+ );let needs_fts = fts_query_probe.is_some();// A `query_string` whose projection DECLINED still has to be// answered by the stored-doc scan — an over-cap `tokens × fields`@@ -15335,7 +15343,12 @@ impl Index {// document happens to sit in. `None` keeps this// segment's own stats (single-arm gate)..with_collection_stats(collection_stats());- let fts_query = query_node_to_fts(query, &text_fields, &exact_fields);+ let fts_query = query_node_to_fts_with_keyword_fields(+ query,+ &text_fields,+ &exact_fields,+ &kw_fields,+ );// Count-only fast path for single-term FTS queries:// call `term_doc_freq` directly on the segment reader@@ -20287,6 +20300,7 @@ impl Index {query: &QueryNode,text_fields: &[String],exact_fields: &HashSet<String>,+ keyword_fields: &HashSet<String>,mem_doc_count: usize,) -> Option<xerj_fts::CollectionStats> {// Widest (field × term) pre-pass we will pay for. A field-less@@ -20297,7 +20311,12 @@ impl Index {// convention as `MAX_QS_CROSS_PRODUCT`).const MAX_STATS_PROBES: usize = 4096;- let fq = query_node_to_fts(query, text_fields, exact_fields)?;+ let fq = query_node_to_fts_with_keyword_fields(+ query,+ text_fields,+ exact_fields,+ keyword_fields,+ )?;let mut fields: Vec<String> = Vec::new();collect_fts_query_fields(&fq, &mut fields);if fields.is_empty() {@@ -29490,8 +29509,10 @@ async fn try_aggs_fast_with_segments(/// Extract a plain-text query string from a QueryNode (for memtable BM25 search).///-/// Only full-text query types are handled here. Term-level queries (Term, Terms,-/// Range, Exists, Prefix, Wildcard) are handled by doc scanning instead.+/// Only full-text query types are handled here. Term-level queries are handled+/// by the memtable's structured paths or doc scanning; scalar keyword terms+/// are kept in the segment FTS bool projection so mixed clauses retain exact+/// membership and scoring there.fn extract_query_text(q: &QueryNode) -> Option<String> {match q {// Wildcard field match must use doc scanning, not BM25.@@ -34594,11 +34615,14 @@ fn query_string_has_no_tokens(/// Convert a QueryNode to an FTS Query for segment search.////// `exact_fields` are the non-Text schema fields (keyword / numeric / date /-/// bool / ip). `build_fts_field_configs` indexes those with the `keyword`+/// bool / ip). `build_fts_field_configs` indexes those with the `keyword`/// analyzer — the whole value becomes ONE case-preserved term — so the query/// side must look them up by whole value too (ES semantics: `match` //// `multi_match` / query-string clauses on a keyword field are exact/// whole-value comparisons, because the keyword analyzer is a no-op).+/// `keyword_fields` is the narrower schema-typed subset that is safe for a+/// scalar `term` projection. Numeric/date/bool/IP terms remain on their+/// source/doc-values path, where typed equality and formatting are preserved./// Tokenizing the query with the `standard` analyzer here would produce/// terms (e.g. "claude" from "claude-haiku-4-5") that can never exist in a/// keyword field's FST — the cause of the multi_match / query_string /@@ -34644,10 +34668,23 @@ fn collect_field_boosts(q: &QueryNode, out: &mut HashMap<String, f32>) {}}+#[cfg(test)]fn query_node_to_fts(q: &QueryNode,text_fields: &[String],exact_fields: &std::collections::HashSet<String>,+) -> Option<FtsQuery> {+ // The projection tests below pass the exact-field set as their keyword+ // subset. The search path uses the schema-aware helper so IP/date/numeric+ // terms retain their source/DV semantics.+ query_node_to_fts_with_keyword_fields(q, text_fields, exact_fields, exact_fields)+}++fn query_node_to_fts_with_keyword_fields(+ q: &QueryNode,+ text_fields: &[String],+ exact_fields: &std::collections::HashSet<String>,+ keyword_fields: &std::collections::HashSet<String>,) -> Option<FtsQuery> {match q {QueryNode::MatchAll => {@@ -34660,7 +34697,9 @@ fn query_node_to_fts(// applied post-hoc by the top-level override in `search` (the// keyword-schema shape is served bit-exactly by `scored_columnar`// before this projection is ever consulted).- QueryNode::Constant { query, .. } => query_node_to_fts(query, text_fields, exact_fields),+ QueryNode::Constant { query, .. } => {+ query_node_to_fts_with_keyword_fields(query, text_fields, exact_fields, keyword_fields)+ }QueryNode::Match {field,query,@@ -34969,17 +35008,36 @@ fn query_node_to_fts(Some(combined)}}- QueryNode::Term { .. } => {- // Term queries are routed through stored-doc scanning- // (json_values_equal) because the FTS index applies the text- // analyzer at index time — so `term {method: "GET"}` against a- // keyword/int/date field would miss (stopword for "GET", wrong- // type for integer). Doc scanning handles all field types- // correctly. Trade-off: slower on huge segments, but segments- // are merged aggressively and most term queries are highly- // selective.- None+ QueryNode::Term {+ field,+ value,+ boost,+ } if keyword_fields.contains(field) && exact_fields.contains(field) => {+ // Declared keyword fields use the keyword analyzer, so a scalar+ // term is already the exact FTS term. Keeping it in the FTS tree+ // is essential for a mixed bool: the FTS bool executor can then+ // union/intersect it with text clauses and add its BM25 score+ // instead of routing the whole tree through the schema-blind+ // stored-source fallback.+ //+ // This does not change the repository's existing post-flush+ // multi-valued-keyword limitation: the segment sidecar flattens+ // source arrays into one keyword token. Array/object query values+ // therefore decline here, and scalar keyword coverage is the+ // deliberately safe projection fixed by this change.+ let term = match value {+ Value::String(s) => s.clone(),+ Value::Number(n) => n.to_string(),+ Value::Bool(b) => b.to_string(),+ Value::Null | Value::Array(_) | Value::Object(_) => return None,+ };+ Some(FtsQuery::Term(FtsTerm::boosted(+ field.as_str(),+ &term,+ boost.unwrap_or(1.0),+ )))}+ QueryNode::Term { .. } => None,QueryNode::Bool {must,should,@@ -35016,10 +35074,15 @@ fn query_node_to_fts(// with keyword-field projections now producing real matches, a// dropped filter would silently overcount. Project them as// `must`, or fall back to the stored scan when one can't- // project (Term/Range/etc. all project to None, so classic- // filters keep taking the doc-scan path as before).+ // project (unsupported Term/Range/etc. still project to None, so+ // classic filters keep taking the doc-scan path as before).for sub in must.iter().chain(filter.iter()) {- let fq = query_node_to_fts(sub, text_fields, exact_fields)?;+ let fq = query_node_to_fts_with_keyword_fields(+ sub,+ text_fields,+ exact_fields,+ keyword_fields,+ )?;bool_q = bool_q.must(fq);projected_any = true;}@@ -35029,14 +35092,24 @@ fn query_node_to_fts(// dropped — dropping it makes the bool LESS permissive and// loses docs that match only via that clause. Fall back to// the stored-doc scan, which handles every child shape.- let fq = query_node_to_fts(sub, text_fields, exact_fields)?;+ let fq = query_node_to_fts_with_keyword_fields(+ sub,+ text_fields,+ exact_fields,+ keyword_fields,+ )?;bool_q = bool_q.should(fq);projected_any = true;}// `must_not` children that don't project are similar: dropping// a must_not relaxes the filter, which is wrong.for sub in must_not {- let fq = query_node_to_fts(sub, text_fields, exact_fields)?;+ let fq = query_node_to_fts_with_keyword_fields(+ sub,+ text_fields,+ exact_fields,+ keyword_fields,+ )?;bool_q = bool_q.must_not(fq);projected_any = true;}@@ -37742,6 +37815,23 @@ mod fts_projection_tests {}}+ #[test]+ fn scalar_term_on_keyword_field_projects_whole_value() {+ let q = QueryNode::Term {+ field: "ax_path".into(),+ value: Value::String("no/such/file.php".into()),+ boost: None,+ };+ let fq = query_node_to_fts(&q, &[], &kw(&["ax_path"])).expect("projects");+ match fq {+ FtsQuery::Term(term) => {+ assert_eq!(term.field, "ax_path");+ assert_eq!(term.term, "no/such/file.php");+ }+ other => panic!("expected keyword term, got {other:?}"),+ }+ }+/// multi_match over keyword fields: one whole-value term per field/// (ES: match on keyword requires whole-value equality per field).#[test]diff --git a/engine/crates/xerj-engine/tests/bool_keyword_term_scoring.rs b/engine/crates/xerj-engine/tests/bool_keyword_term_scoring.rsnew file mode 100644index 000000000..50e7cb6ea--- /dev/null+++ b/engine/crates/xerj-engine/tests/bool_keyword_term_scoring.rs@@ -0,0 +1,278 @@+//! Regression tests for keyword `term` clauses inside scored `bool` queries.+//!+//! A keyword term used to make the whole bool decline the FTS projection and+//! fall back to the schema-blind stored-source matcher. That fallback split+//! identifiers such as `is_wp_error` into three OR tokens and used its flat+//! fallback score, so a zero-matching keyword term changed both membership and+//! scores of an unrelated text clause.++use std::collections::{BTreeMap, BTreeSet};++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::executor::SearchResult;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn request(query: Value) -> xerj_query::ast::SearchRequest {+ parse_request(&json!({"query": query, "size": 20})).expect("parse_request")+}++async fn seed() -> (TempDir, std::sync::Arc<Index>) {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("body", FieldType::Text));+ schema+ .fields+ .push(FieldConfig::new("defs", FieldType::Text));+ schema+ .fields+ .push(FieldConfig::new("ax_path", FieldType::Keyword));+ engine.create_index("bool-term", schema).unwrap();+ let idx = engine.get_index("bool-term").unwrap();++ idx.index_document(+ Some("hit".into()),+ json!({+ "body": "is_wp_error",+ "defs": "is_wp_error",+ "ax_path": "wp/hit.php"+ }),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("is-only".into()),+ json!({"body": "unrelated", "defs": "is", "ax_path": "wp/is.php"}),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("wp-only".into()),+ json!({"body": "unrelated", "defs": "wp", "ax_path": "wp/wp.php"}),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("error-only".into()),+ json!({"body": "unrelated", "defs": "error", "ax_path": "wp/error.php"}),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("neither".into()),+ json!({+ "body": "unrelated",+ "defs": "unrelated",+ "ax_path": "wp/neither.php"+ }),+ )+ .await+ .unwrap();++ // The standalone text queries use the segment FTS path. The bools under+ // test take a different path on the base commit, so flushing is required+ // to expose the divergence deterministically.+ idx.flush().await.unwrap();+ (dir, idx)+}++fn ids(result: &SearchResult) -> BTreeSet<String> {+ result.hits.iter().map(|hit| hit.id.clone()).collect()+}++fn scores(result: &SearchResult) -> BTreeMap<String, f32> {+ result+ .hits+ .iter()+ .map(|hit| (hit.id.clone(), hit.score))+ .collect()+}++fn assert_same_hits(label: &str, expected: &SearchResult, actual: &SearchResult) {+ assert_eq!(+ actual.total.value, expected.total.value,+ "{label}: total changed"+ );+ assert_eq!(ids(actual), ids(expected), "{label}: matched IDs changed");++ let expected_scores = scores(expected);+ let actual_scores = scores(actual);+ for (id, expected_score) in expected_scores {+ let actual_score = actual_scores+ .get(&id)+ .unwrap_or_else(|| panic!("{label}: missing score for {id}"));+ assert!(+ (actual_score - expected_score).abs() < 1e-5,+ "{label}: score for {id} changed from {expected_score} to {actual_score}"+ );+ }+}++#[tokio::test]+async fn zero_keyword_term_does_not_change_multimatch_should() {+ let (_dir, idx) = seed().await;+ let baseline = idx+ .search(&request(json!({+ "multi_match": {+ "query": "is_wp_error",+ "fields": ["body", "defs"],+ "type": "most_fields"+ }+ })))+ .await+ .unwrap();+ let with_zero_term = idx+ .search(&request(json!({+ "bool": {+ "should": [+ {"multi_match": {+ "query": "is_wp_error",+ "fields": ["body", "defs"],+ "type": "most_fields"+ }},+ {"term": {"ax_path": "no/such/file.php"}}+ ]+ }+ })))+ .await+ .unwrap();++ assert_eq!(baseline.total.value, 1, "fixture baseline must be one hit");+ assert!(+ baseline.hits[0].score > 1.0,+ "fixture must expose the lost scored path, got {}",+ baseline.hits[0].score+ );+ assert_same_hits("multi_match + zero term", &baseline, &with_zero_term);+}++#[tokio::test]+async fn zero_keyword_term_does_not_change_match_should() {+ let (_dir, idx) = seed().await;+ let baseline = idx+ .search(&request(json!({"match": {"defs": "is_wp_error"}})))+ .await+ .unwrap();+ let with_zero_term = idx+ .search(&request(json!({+ "bool": {+ "should": [+ {"match": {"defs": "is_wp_error"}},+ {"term": {"ax_path": "no/such/file.php"}}+ ]+ }+ })))+ .await+ .unwrap();++ assert_eq!(baseline.total.value, 1, "fixture baseline must be one hit");+ assert_same_hits("match + zero term", &baseline, &with_zero_term);+}++#[tokio::test]+async fn zero_keyword_term_does_not_expand_match_must() {+ let (_dir, idx) = seed().await;+ let baseline = idx+ .search(&request(json!({"match": {"defs": "is_wp_error"}})))+ .await+ .unwrap();+ let with_zero_term = idx+ .search(&request(json!({+ "bool": {+ "must": [{"match": {"defs": "is_wp_error"}}],+ "should": [{"term": {"ax_path": "no/such/file.php"}}]+ }+ })))+ .await+ .unwrap();++ assert_same_hits("must match + zero term", &baseline, &with_zero_term);+}++#[tokio::test]+async fn minimum_should_match_rejects_a_zero_matching_keyword_term() {+ let (_dir, idx) = seed().await;+ let result = idx+ .search(&request(json!({+ "bool": {+ "must": [{"match": {"defs": "is_wp_error"}}],+ "should": [{"term": {"ax_path": "no/such/file.php"}}],+ "minimum_should_match": 1+ }+ })))+ .await+ .unwrap();++ assert_eq!(result.total.value, 0);+ assert!(result.hits.is_empty());+}++#[tokio::test]+async fn matching_keyword_term_adds_score_and_union_membership() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("defs", FieldType::Text));+ schema+ .fields+ .push(FieldConfig::new("ax_path", FieldType::Keyword));+ engine.create_index("bool-term-additive", schema).unwrap();+ let idx = engine.get_index("bool-term-additive").unwrap();+ for (id, defs, path) in [+ ("both", "is_wp_error", "same/path.php"),+ ("text", "is_wp_error", "other/path.php"),+ ("term", "unrelated", "same/path.php"),+ ("neither", "unrelated", "other/path.php"),+ ] {+ idx.index_document(Some(id.into()), json!({"defs": defs, "ax_path": path}))+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();++ let text = idx+ .search(&request(json!({"match": {"defs": "is_wp_error"}})))+ .await+ .unwrap();+ let keyword = idx+ .search(&request(json!({"term": {"ax_path": "same/path.php"}})))+ .await+ .unwrap();+ let combined = idx+ .search(&request(json!({+ "bool": {+ "should": [+ {"match": {"defs": "is_wp_error"}},+ {"term": {"ax_path": "same/path.php"}}+ ]+ }+ })))+ .await+ .unwrap();++ assert_eq!(+ ids(&combined),+ BTreeSet::from(["both".into(), "text".into(), "term".into()])+ );+ let text_score = scores(&text)["both"];+ let keyword_score = scores(&keyword)["both"];+ let combined_score = scores(&combined)["both"];+ assert!(+ (combined_score - (text_score + keyword_score)).abs() < 1e-5,+ "combined score {combined_score} did not add text {text_score} + keyword {keyword_score}"+ );+}