AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): term/match_phrase queries on boolean fields undercounted to 0
Vincenzo Lombardo7edc992bc1b8
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0233b97b7..23d742477 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -13051,8 +13051,26 @@ impl Index {if self.memtable.doc_count() == 0 {0} else {- let target_str: Option<String> = value.as_str().map(String::from);- let target_num: Option<f64> = value.as_f64();+ // `value.as_str()`/`.as_f64()` both return `None` for a+ // JSON boolean, so a boolean-field term query previously+ // never matched a single memtable-resident doc regardless+ // of actual content — found empirically: `mem_doc_count`+ // was non-zero (this branch DOES run) yet a real boolean+ // filter still undercounted to a confident, wrong 0. Match+ // `scored_fast_plan`'s `scoring_leaf` (already correct for+ // this exact case) and the memtable's own on-insert+ // encoding (`push_field`'s `Value::Bool` arm, which stores+ // BOTH the 1.0/0.0 numeric form and the "true"/"false"+ // keyword form) by coercing bool → both representations+ // here too.+ let target_str: Option<String> = match value {+ Value::Bool(b) => Some(b.to_string()),+ _ => value.as_str().map(String::from),+ };+ let target_num: Option<f64> = match value {+ Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),+ _ => value.as_f64(),+ };let mut m = 0u64;if let Some(tgt) = target_num {m += self.memtable.doc_values_numeric_count(field, tgt) as u64;@@ -13089,7 +13107,17 @@ impl Index {other => other.to_string(),};let lowered = raw.to_ascii_lowercase();- let target_num: Option<f64> = value.as_f64();+ // Same bool coercion as the memtable side above — `value.as_f64()`+ // alone returns `None` for a JSON boolean, which used to make+ // `served_by_dv` false for every segment whose on-disk column for+ // this field is `Column::Numeric` (the boolean encoding+ // `build_doc_value_columns` always uses), falling through to an+ // FTS lookup that has no real content for a non-analyzed boolean+ // field and confidently returns 0 instead of the correct count.+ let target_num: Option<f64> = match value {+ Value::Bool(b) => Some(if *b { 1.0 } else { 0.0 }),+ _ => value.as_f64(),+ };let mut seg_matches: u64 = 0;for meta in &snap.segments {@@ -21093,7 +21121,25 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {} => {get_field_value(source, field).and_then(|v| match v {- Value::String(s) => {+ // Real ES tokenizes whatever the field holds — a+ // `boolean`/numeric field's value is just its string+ // form ("true"/"false", "42") for phrase-matching+ // purposes. Pre-fix this arm only handled+ // `Value::String`, so a `match_phrase` against a+ // boolean field's `_source` value (a genuine JSON+ // `Bool`, never a `String`) fell to the `_ => None`+ // catch-all and NEVER matched any document, regardless+ // of value — found empirically alongside the parser-+ // level scalar-coercion fix for the *query* side+ // (xerj-query's `match_phrase`/`match_phrase_prefix`):+ // that fix stops the crash on `{query: true}`, but+ // without this one the (now-valid) query still+ // silently matched 0 documents.+ Value::String(_) | Value::Bool(_) | Value::Number(_) => {+ let s = match v {+ Value::String(s) => s.clone(),+ other => other.to_string(),+ };let field_tokens: Vec<String> = s.to_lowercase().split(|c: char| !c.is_alphanumeric())diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex da8e2e6ec..681717f46 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -419,6 +419,28 @@ impl DocValues {}}Value::Bool(b) => {+ // Numeric column FIRST, matching the on-disk segment+ // encoding (`build_doc_value_columns` in index.rs: a+ // boolean is always the f64 bit-pattern of 1.0/0.0, never+ // a keyword-only value) — the disk-segment builder reads+ // straight from the same `Value` tree and has always+ // gotten this right, but the memtable's own in-memory+ // representation never populated `numeric` for booleans at+ // all, only `keyword`. Any query still hitting a+ // memtable-resident boolean doc through a numeric-column+ // path (`ScoreEval::Bool`, `doc_values_numeric_count`,+ // range/terms-agg fast paths) saw the field as absent —+ // found empirically: a real OpenSearch Dashboards filter+ // pill on a boolean field returned 0 hits against+ // freshly-bulk-imported data (matches every+ // real-world/OSD import shape — natural background flush,+ // no explicit `_flush` — while the aggregation reading+ // the SAME field correctly bucketed true/false, since+ // aggregations use a different, disk-segment-only read+ // path that was never affected).+ let ncol = entry_no_clone(&mut self.numeric, field, Default::default);+ pad_to(ncol, doc_index);+ ncol.push(Some(if *b { 1.0 } else { 0.0 }));let kcol = entry_no_clone(&mut self.keyword, field, Default::default);pad_to(kcol, doc_index);kcol.push(Some(b.to_string()));