Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): term/match_phrase queries on boolean fields undercounted to 0

Vincenzo Lombardo7edc992bc1b8
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0233b97b7..23d742477 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -13051,8 +13051,26 @@ impl Index {
if self.memtable.doc_count() == 0 {
0
} else {
- let target_str: Option<String> = value.as_str().map(String::from);
- let target_num: Option<f64> = value.as_f64();
+ // `value.as_str()`/`.as_f64()` both return `None` for a
+ // JSON boolean, so a boolean-field term query previously
+ // never matched a single memtable-resident doc regardless
+ // of actual content — found empirically: `mem_doc_count`
+ // was non-zero (this branch DOES run) yet a real boolean
+ // filter still undercounted to a confident, wrong 0. Match
+ // `scored_fast_plan`'s `scoring_leaf` (already correct for
+ // this exact case) and the memtable's own on-insert
+ // encoding (`push_field`'s `Value::Bool` arm, which stores
+ // BOTH the 1.0/0.0 numeric form and the "true"/"false"
+ // keyword form) by coercing bool → both representations
+ // here too.
+ let target_str: Option<String> = match value {
+ Value::Bool(b) => Some(b.to_string()),
+ _ => value.as_str().map(String::from),
+ };
+ let target_num: Option<f64> = match value {
+ Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),
+ _ => value.as_f64(),
+ };
let mut m = 0u64;
if let Some(tgt) = target_num {
m += self.memtable.doc_values_numeric_count(field, tgt) as u64;
@@ -13089,7 +13107,17 @@ impl Index {
other => other.to_string(),
};
let lowered = raw.to_ascii_lowercase();
- let target_num: Option<f64> = value.as_f64();
+ // Same bool coercion as the memtable side above — `value.as_f64()`
+ // alone returns `None` for a JSON boolean, which used to make
+ // `served_by_dv` false for every segment whose on-disk column for
+ // this field is `Column::Numeric` (the boolean encoding
+ // `build_doc_value_columns` always uses), falling through to an
+ // FTS lookup that has no real content for a non-analyzed boolean
+ // field and confidently returns 0 instead of the correct count.
+ let target_num: Option<f64> = match value {
+ Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),
+ _ => value.as_f64(),
+ };
let mut seg_matches: u64 = 0;
for meta in &snap.segments {
@@ -21093,7 +21121,25 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
} => {
get_field_value(source, field)
.and_then(|v| match v {
- Value::String(s) => {
+ // Real ES tokenizes whatever the field holds — a
+ // `boolean`/numeric field's value is just its string
+ // form ("true"/"false", "42") for phrase-matching
+ // purposes. Pre-fix this arm only handled
+ // `Value::String`, so a `match_phrase` against a
+ // boolean field's `_source` value (a genuine JSON
+ // `Bool`, never a `String`) fell to the `_ => None`
+ // catch-all and NEVER matched any document, regardless
+ // of value — found empirically alongside the parser-
+ // level scalar-coercion fix for the *query* side
+ // (xerj-query's `match_phrase`/`match_phrase_prefix`):
+ // that fix stops the crash on `{query: true}`, but
+ // without this one the (now-valid) query still
+ // silently matched 0 documents.
+ Value::String(_) | Value::Bool(_) | Value::Number(_) => {
+ let s = match v {
+ Value::String(s) => s.clone(),
+ other => other.to_string(),
+ };
let field_tokens: Vec<String> = s
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index da8e2e6ec..681717f46 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -419,6 +419,28 @@ impl DocValues {
}
}
Value::Bool(b) => {
+ // Numeric column FIRST, matching the on-disk segment
+ // encoding (`build_doc_value_columns` in index.rs: a
+ // boolean is always the f64 bit-pattern of 1.0/0.0, never
+ // a keyword-only value) — the disk-segment builder reads
+ // straight from the same `Value` tree and has always
+ // gotten this right, but the memtable's own in-memory
+ // representation never populated `numeric` for booleans at
+ // all, only `keyword`. Any query still hitting a
+ // memtable-resident boolean doc through a numeric-column
+ // path (`ScoreEval::Bool`, `doc_values_numeric_count`,
+ // range/terms-agg fast paths) saw the field as absent —
+ // found empirically: a real OpenSearch Dashboards filter
+ // pill on a boolean field returned 0 hits against
+ // freshly-bulk-imported data (matches every
+ // real-world/OSD import shape — natural background flush,
+ // no explicit `_flush` — while the aggregation reading
+ // the SAME field correctly bucketed true/false, since
+ // aggregations use a different, disk-segment-only read
+ // path that was never affected).
+ let ncol = entry_no_clone(&mut self.numeric, field, Default::default);
+ pad_to(ncol, doc_index);
+ ncol.push(Some(if *b { 1.0 } else { 0.0 }));
let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
pad_to(kcol, doc_index);
kcol.push(Some(b.to_string()));