Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): preserve bool keyword-term scoring

buger4683fe2acd9b
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 2187c51d7..f5938cc63 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -14255,6 +14255,7 @@ impl Index {
query,
&text_fields,
&exact_fields,
+ &kw_fields,
mem_doc_count,
)
.map(Arc::new)
@@ -14878,7 +14879,9 @@ impl Index {
// and materialises only the top prefix, so the F1 bounded-scan / count
// overwrite must NOT touch FTS queries — it only applies to the
// non-FTS stored-doc scan (match_all / term-on-keyword / range).
- let query_needs_fts: bool = query_node_to_fts(query, &text_fields, &exact_fields).is_some();
+ let query_needs_fts: bool =
+ query_node_to_fts_with_keyword_fields(query, &text_fields, &exact_fields, &kw_fields)
+ .is_some();
// ── Precomputed segment agg fast path (M2 G2) ─────────────────────
//
@@ -15122,7 +15125,12 @@ impl Index {
// entirely — that call eagerly reads .fst/.meta/.post/.norms
// for every text field, which costs ~50 MB per segment and is
// wasted work when the query is a stored-doc scan.
- let fts_query_probe = query_node_to_fts(query, &text_fields, &exact_fields);
+ let fts_query_probe = query_node_to_fts_with_keyword_fields(
+ query,
+ &text_fields,
+ &exact_fields,
+ &kw_fields,
+ );
let needs_fts = fts_query_probe.is_some();
// A `query_string` whose projection DECLINED still has to be
// answered by the stored-doc scan — an over-cap `tokens × fields`
@@ -15335,7 +15343,12 @@ impl Index {
// document happens to sit in. `None` keeps this
// segment's own stats (single-arm gate).
.with_collection_stats(collection_stats());
- let fts_query = query_node_to_fts(query, &text_fields, &exact_fields);
+ let fts_query = query_node_to_fts_with_keyword_fields(
+ query,
+ &text_fields,
+ &exact_fields,
+ &kw_fields,
+ );
// Count-only fast path for single-term FTS queries:
// call `term_doc_freq` directly on the segment reader
@@ -20287,6 +20300,7 @@ impl Index {
query: &QueryNode,
text_fields: &[String],
exact_fields: &HashSet<String>,
+ keyword_fields: &HashSet<String>,
mem_doc_count: usize,
) -> Option<xerj_fts::CollectionStats> {
// Widest (field × term) pre-pass we will pay for. A field-less
@@ -20297,7 +20311,12 @@ impl Index {
// convention as `MAX_QS_CROSS_PRODUCT`).
const MAX_STATS_PROBES: usize = 4096;
- let fq = query_node_to_fts(query, text_fields, exact_fields)?;
+ let fq = query_node_to_fts_with_keyword_fields(
+ query,
+ text_fields,
+ exact_fields,
+ keyword_fields,
+ )?;
let mut fields: Vec<String> = Vec::new();
collect_fts_query_fields(&fq, &mut fields);
if fields.is_empty() {
@@ -29490,8 +29509,10 @@ async fn try_aggs_fast_with_segments(
/// Extract a plain-text query string from a QueryNode (for memtable BM25 search).
///
-/// Only full-text query types are handled here. Term-level queries (Term, Terms,
-/// Range, Exists, Prefix, Wildcard) are handled by doc scanning instead.
+/// Only full-text query types are handled here. Term-level queries are handled
+/// by the memtable's structured paths or doc scanning; scalar keyword terms
+/// are kept in the segment FTS bool projection so mixed clauses retain exact
+/// membership and scoring there.
fn extract_query_text(q: &QueryNode) -> Option<String> {
match q {
// Wildcard field match must use doc scanning, not BM25.
@@ -34594,11 +34615,14 @@ fn query_string_has_no_tokens(
/// Convert a QueryNode to an FTS Query for segment search.
///
/// `exact_fields` are the non-Text schema fields (keyword / numeric / date /
-/// bool / ip). `build_fts_field_configs` indexes those with the `keyword`
+/// bool / ip). `build_fts_field_configs` indexes those with the `keyword`
/// analyzer — the whole value becomes ONE case-preserved term — so the query
/// side must look them up by whole value too (ES semantics: `match` /
/// `multi_match` / query-string clauses on a keyword field are exact
/// whole-value comparisons, because the keyword analyzer is a no-op).
+/// `keyword_fields` is the narrower schema-typed subset that is safe for a
+/// scalar `term` projection. Numeric/date/bool/IP terms remain on their
+/// source/doc-values path, where typed equality and formatting are preserved.
/// Tokenizing the query with the `standard` analyzer here would produce
/// terms (e.g. "claude" from "claude-haiku-4-5") that can never exist in a
/// keyword field's FST — the cause of the multi_match / query_string /
@@ -34644,10 +34668,23 @@ fn collect_field_boosts(q: &QueryNode, out: &mut HashMap<String, f32>) {
}
}
+#[cfg(test)]
fn query_node_to_fts(
q: &QueryNode,
text_fields: &[String],
exact_fields: &std::collections::HashSet<String>,
+) -> Option<FtsQuery> {
+ // The projection tests below pass the exact-field set as their keyword
+ // subset. The search path uses the schema-aware helper so IP/date/numeric
+ // terms retain their source/DV semantics.
+ query_node_to_fts_with_keyword_fields(q, text_fields, exact_fields, exact_fields)
+}
+
+fn query_node_to_fts_with_keyword_fields(
+ q: &QueryNode,
+ text_fields: &[String],
+ exact_fields: &std::collections::HashSet<String>,
+ keyword_fields: &std::collections::HashSet<String>,
) -> Option<FtsQuery> {
match q {
QueryNode::MatchAll => {
@@ -34660,7 +34697,9 @@ fn query_node_to_fts(
// applied post-hoc by the top-level override in `search` (the
// keyword-schema shape is served bit-exactly by `scored_columnar`
// before this projection is ever consulted).
- QueryNode::Constant { query, .. } => query_node_to_fts(query, text_fields, exact_fields),
+ QueryNode::Constant { query, .. } => {
+ query_node_to_fts_with_keyword_fields(query, text_fields, exact_fields, keyword_fields)
+ }
QueryNode::Match {
field,
query,
@@ -34969,17 +35008,36 @@ fn query_node_to_fts(
Some(combined)
}
}
- QueryNode::Term { .. } => {
- // Term queries are routed through stored-doc scanning
- // (json_values_equal) because the FTS index applies the text
- // analyzer at index time — so `term {method: "GET"}` against a
- // keyword/int/date field would miss (stopword for "GET", wrong
- // type for integer). Doc scanning handles all field types
- // correctly. Trade-off: slower on huge segments, but segments
- // are merged aggressively and most term queries are highly
- // selective.
- None
+ QueryNode::Term {
+ field,
+ value,
+ boost,
+ } if keyword_fields.contains(field) && exact_fields.contains(field) => {
+ // Declared keyword fields use the keyword analyzer, so a scalar
+ // term is already the exact FTS term. Keeping it in the FTS tree
+ // is essential for a mixed bool: the FTS bool executor can then
+ // union/intersect it with text clauses and add its BM25 score
+ // instead of routing the whole tree through the schema-blind
+ // stored-source fallback.
+ //
+ // This does not change the repository's existing post-flush
+ // multi-valued-keyword limitation: the segment sidecar flattens
+ // source arrays into one keyword token. Array/object query values
+ // therefore decline here, and scalar keyword coverage is the
+ // deliberately safe projection fixed by this change.
+ let term = match value {
+ Value::String(s) => s.clone(),
+ Value::Number(n) => n.to_string(),
+ Value::Bool(b) => b.to_string(),
+ Value::Null | Value::Array(_) | Value::Object(_) => return None,
+ };
+ Some(FtsQuery::Term(FtsTerm::boosted(
+ field.as_str(),
+ &term,
+ boost.unwrap_or(1.0),
+ )))
}
+ QueryNode::Term { .. } => None,
QueryNode::Bool {
must,
should,
@@ -35016,10 +35074,15 @@ fn query_node_to_fts(
// with keyword-field projections now producing real matches, a
// dropped filter would silently overcount. Project them as
// `must`, or fall back to the stored scan when one can't
- // project (Term/Range/etc. all project to None, so classic
- // filters keep taking the doc-scan path as before).
+ // project (unsupported Term/Range/etc. still project to None, so
+ // classic filters keep taking the doc-scan path as before).
for sub in must.iter().chain(filter.iter()) {
- let fq = query_node_to_fts(sub, text_fields, exact_fields)?;
+ let fq = query_node_to_fts_with_keyword_fields(
+ sub,
+ text_fields,
+ exact_fields,
+ keyword_fields,
+ )?;
bool_q = bool_q.must(fq);
projected_any = true;
}
@@ -35029,14 +35092,24 @@ fn query_node_to_fts(
// dropped — dropping it makes the bool LESS permissive and
// loses docs that match only via that clause. Fall back to
// the stored-doc scan, which handles every child shape.
- let fq = query_node_to_fts(sub, text_fields, exact_fields)?;
+ let fq = query_node_to_fts_with_keyword_fields(
+ sub,
+ text_fields,
+ exact_fields,
+ keyword_fields,
+ )?;
bool_q = bool_q.should(fq);
projected_any = true;
}
// `must_not` children that don't project are similar: dropping
// a must_not relaxes the filter, which is wrong.
for sub in must_not {
- let fq = query_node_to_fts(sub, text_fields, exact_fields)?;
+ let fq = query_node_to_fts_with_keyword_fields(
+ sub,
+ text_fields,
+ exact_fields,
+ keyword_fields,
+ )?;
bool_q = bool_q.must_not(fq);
projected_any = true;
}
@@ -37742,6 +37815,23 @@ mod fts_projection_tests {
}
}
+ #[test]
+ fn scalar_term_on_keyword_field_projects_whole_value() {
+ let q = QueryNode::Term {
+ field: "ax_path".into(),
+ value: Value::String("no/such/file.php".into()),
+ boost: None,
+ };
+ let fq = query_node_to_fts(&q, &[], &kw(&["ax_path"])).expect("projects");
+ match fq {
+ FtsQuery::Term(term) => {
+ assert_eq!(term.field, "ax_path");
+ assert_eq!(term.term, "no/such/file.php");
+ }
+ other => panic!("expected keyword term, got {other:?}"),
+ }
+ }
+
/// multi_match over keyword fields: one whole-value term per field
/// (ES: match on keyword requires whole-value equality per field).
#[test]
diff --git a/engine/crates/xerj-engine/tests/bool_keyword_term_scoring.rs b/engine/crates/xerj-engine/tests/bool_keyword_term_scoring.rs
new file mode 100644
index 000000000..50e7cb6ea
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/bool_keyword_term_scoring.rs
@@ -0,0 +1,278 @@
+//! Regression tests for keyword `term` clauses inside scored `bool` queries.
+//!
+//! A keyword term used to make the whole bool decline the FTS projection and
+//! fall back to the schema-blind stored-source matcher. That fallback split
+//! identifiers such as `is_wp_error` into three OR tokens and used its flat
+//! fallback score, so a zero-matching keyword term changed both membership and
+//! scores of an unrelated text clause.
+
+use std::collections::{BTreeMap, BTreeSet};
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::executor::SearchResult;
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn request(query: Value) -> xerj_query::ast::SearchRequest {
+ parse_request(&json!({"query": query, "size": 20})).expect("parse_request")
+}
+
+async fn seed() -> (TempDir, std::sync::Arc<Index>) {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("defs", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("ax_path", FieldType::Keyword));
+ engine.create_index("bool-term", schema).unwrap();
+ let idx = engine.get_index("bool-term").unwrap();
+
+ idx.index_document(
+ Some("hit".into()),
+ json!({
+ "body": "is_wp_error",
+ "defs": "is_wp_error",
+ "ax_path": "wp/hit.php"
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("is-only".into()),
+ json!({"body": "unrelated", "defs": "is", "ax_path": "wp/is.php"}),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("wp-only".into()),
+ json!({"body": "unrelated", "defs": "wp", "ax_path": "wp/wp.php"}),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("error-only".into()),
+ json!({"body": "unrelated", "defs": "error", "ax_path": "wp/error.php"}),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("neither".into()),
+ json!({
+ "body": "unrelated",
+ "defs": "unrelated",
+ "ax_path": "wp/neither.php"
+ }),
+ )
+ .await
+ .unwrap();
+
+ // The standalone text queries use the segment FTS path. The bools under
+ // test take a different path on the base commit, so flushing is required
+ // to expose the divergence deterministically.
+ idx.flush().await.unwrap();
+ (dir, idx)
+}
+
+fn ids(result: &SearchResult) -> BTreeSet<String> {
+ result.hits.iter().map(|hit| hit.id.clone()).collect()
+}
+
+fn scores(result: &SearchResult) -> BTreeMap<String, f32> {
+ result
+ .hits
+ .iter()
+ .map(|hit| (hit.id.clone(), hit.score))
+ .collect()
+}
+
+fn assert_same_hits(label: &str, expected: &SearchResult, actual: &SearchResult) {
+ assert_eq!(
+ actual.total.value, expected.total.value,
+ "{label}: total changed"
+ );
+ assert_eq!(ids(actual), ids(expected), "{label}: matched IDs changed");
+
+ let expected_scores = scores(expected);
+ let actual_scores = scores(actual);
+ for (id, expected_score) in expected_scores {
+ let actual_score = actual_scores
+ .get(&id)
+ .unwrap_or_else(|| panic!("{label}: missing score for {id}"));
+ assert!(
+ (actual_score - expected_score).abs() < 1e-5,
+ "{label}: score for {id} changed from {expected_score} to {actual_score}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn zero_keyword_term_does_not_change_multimatch_should() {
+ let (_dir, idx) = seed().await;
+ let baseline = idx
+ .search(&request(json!({
+ "multi_match": {
+ "query": "is_wp_error",
+ "fields": ["body", "defs"],
+ "type": "most_fields"
+ }
+ })))
+ .await
+ .unwrap();
+ let with_zero_term = idx
+ .search(&request(json!({
+ "bool": {
+ "should": [
+ {"multi_match": {
+ "query": "is_wp_error",
+ "fields": ["body", "defs"],
+ "type": "most_fields"
+ }},
+ {"term": {"ax_path": "no/such/file.php"}}
+ ]
+ }
+ })))
+ .await
+ .unwrap();
+
+ assert_eq!(baseline.total.value, 1, "fixture baseline must be one hit");
+ assert!(
+ baseline.hits[0].score > 1.0,
+ "fixture must expose the lost scored path, got {}",
+ baseline.hits[0].score
+ );
+ assert_same_hits("multi_match + zero term", &baseline, &with_zero_term);
+}
+
+#[tokio::test]
+async fn zero_keyword_term_does_not_change_match_should() {
+ let (_dir, idx) = seed().await;
+ let baseline = idx
+ .search(&request(json!({"match": {"defs": "is_wp_error"}})))
+ .await
+ .unwrap();
+ let with_zero_term = idx
+ .search(&request(json!({
+ "bool": {
+ "should": [
+ {"match": {"defs": "is_wp_error"}},
+ {"term": {"ax_path": "no/such/file.php"}}
+ ]
+ }
+ })))
+ .await
+ .unwrap();
+
+ assert_eq!(baseline.total.value, 1, "fixture baseline must be one hit");
+ assert_same_hits("match + zero term", &baseline, &with_zero_term);
+}
+
+#[tokio::test]
+async fn zero_keyword_term_does_not_expand_match_must() {
+ let (_dir, idx) = seed().await;
+ let baseline = idx
+ .search(&request(json!({"match": {"defs": "is_wp_error"}})))
+ .await
+ .unwrap();
+ let with_zero_term = idx
+ .search(&request(json!({
+ "bool": {
+ "must": [{"match": {"defs": "is_wp_error"}}],
+ "should": [{"term": {"ax_path": "no/such/file.php"}}]
+ }
+ })))
+ .await
+ .unwrap();
+
+ assert_same_hits("must match + zero term", &baseline, &with_zero_term);
+}
+
+#[tokio::test]
+async fn minimum_should_match_rejects_a_zero_matching_keyword_term() {
+ let (_dir, idx) = seed().await;
+ let result = idx
+ .search(&request(json!({
+ "bool": {
+ "must": [{"match": {"defs": "is_wp_error"}}],
+ "should": [{"term": {"ax_path": "no/such/file.php"}}],
+ "minimum_should_match": 1
+ }
+ })))
+ .await
+ .unwrap();
+
+ assert_eq!(result.total.value, 0);
+ assert!(result.hits.is_empty());
+}
+
+#[tokio::test]
+async fn matching_keyword_term_adds_score_and_union_membership() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("defs", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("ax_path", FieldType::Keyword));
+ engine.create_index("bool-term-additive", schema).unwrap();
+ let idx = engine.get_index("bool-term-additive").unwrap();
+ for (id, defs, path) in [
+ ("both", "is_wp_error", "same/path.php"),
+ ("text", "is_wp_error", "other/path.php"),
+ ("term", "unrelated", "same/path.php"),
+ ("neither", "unrelated", "other/path.php"),
+ ] {
+ idx.index_document(Some(id.into()), json!({"defs": defs, "ax_path": path}))
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ let text = idx
+ .search(&request(json!({"match": {"defs": "is_wp_error"}})))
+ .await
+ .unwrap();
+ let keyword = idx
+ .search(&request(json!({"term": {"ax_path": "same/path.php"}})))
+ .await
+ .unwrap();
+ let combined = idx
+ .search(&request(json!({
+ "bool": {
+ "should": [
+ {"match": {"defs": "is_wp_error"}},
+ {"term": {"ax_path": "same/path.php"}}
+ ]
+ }
+ })))
+ .await
+ .unwrap();
+
+ assert_eq!(
+ ids(&combined),
+ BTreeSet::from(["both".into(), "text".into(), "term".into()])
+ );
+ let text_score = scores(&text)["both"];
+ let keyword_score = scores(&keyword)["both"];
+ let combined_score = scores(&combined)["both"];
+ assert!(
+ (combined_score - (text_score + keyword_score)).abs() < 1e-5,
+ "combined score {combined_score} did not add text {text_score} + keyword {keyword_score}"
+ );
+}