AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(aggs): real sum_other_doc_count for terms + multi_terms on every path (RC4-W2 item 5)
xerj-orgf1a4a5cab942
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 38e891c12..13f4be851 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -9,7 +9,8 @@//!//! Concrete implications://! - `terms` bucket `doc_count` is always precise — `doc_count_error_upper_bound`-//! is always 0 and `sum_other_doc_count` is 0 for results within `size`.+//! is always 0, and `sum_other_doc_count` is the exact total of the doc+//! counts truncated away by `size`/`shard_size` (0 when nothing was cut).//! - `cardinality` returns the true distinct count, not an HLL estimate.//! - There is no `"relation": "gte"` approximation on total hits.//!@@ -2685,14 +2686,15 @@ fn run_terms(.collect()});- // Apply min_doc_count filter before sorting — ES default is 1, tests set- // values as low as 0 (to force empty-term buckets into the output).+ // Collection-time filters only (partition + include/exclude) — these+ // terms never existed as buckets, so they don't count toward+ // `sum_other_doc_count` (verified vs live ES). The `min_doc_count`+ // filter is NOT applied here: ES applies it at final reduce, after the+ // shard-size cut — `apply_terms_size_pipeline` below reproduces that+ // ordering so both the bucket set and the "other" count match ES.let mut candidates: Vec<(String, u64)> = counts.into_iter()- .filter(|(k, c)| {- if *c < min_doc_count {- return false;- }+ .filter(|(k, _c)| {if let (Some(p), Some(n)) = (partition, num_partitions) {if n > 0 {// ES uses Murmur3_32 (`StringHelper.murmurhash3_x86_32`)@@ -2773,9 +2775,15 @@ fn run_terms(let mut sorted: Vec<(String, u64, Option<Value>)> = pre_computed;sorted.sort_by(|a, b| cmp_terms_by_orders(a, b, &orders));- if let Some(n) = cap {- sorted.truncate(n);- }+ // ES single-shard truncation accounting: shard-size cut →+ // min_doc_count reduce-drop → size cut; cut/overflowed bucket counts+ // accumulate into the REAL sum_other_doc_count (previously hardcoded 0).+ let shard_size_param: Option<usize> = params+ .get("shard_size")+ .and_then(Value::as_u64)+ .map(|v| v as usize);+ let (sorted, sum_other_doc_count) =+ apply_terms_size_pipeline(sorted, |r| r.1, cap, shard_size_param, min_doc_count);let buckets: Vec<Value> = sorted.into_iter()@@ -2827,11 +2835,70 @@ fn run_terms(json!({"doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0,+ "sum_other_doc_count": sum_other_doc_count,"buckets": buckets})}+/// Apply Elasticsearch's (single-shard) terms-agg size pipeline to an+/// already-ordered candidate list and compute the real+/// `sum_other_doc_count`. Semantics verified against live ES 8.13.4:+///+/// 1. Shard select — the shard keeps the top `shard_size` buckets in the+/// requested order (default `size*1.5 + 10`, and never below `size`,+/// matching `BucketUtils.suggestShardSideQueueSize` + ES's+/// shard_size >= size reset). Buckets cut here count toward+/// `sum_other_doc_count`.+/// 2. Final reduce — buckets failing `min_doc_count` are dropped+/// SILENTLY (their counts do NOT go to "other"; verified live);+/// passing buckets beyond `size` overflow into "other".+///+/// `cap == None` is xerj's `size: 0` extension (return every bucket): no+/// truncation happens, only the `min_doc_count` filter, and "other" is 0.+///+/// Shared by `run_terms`, `run_multi_terms`, the columnar+/// `fast_aggs::exec_terms` arm and the doc-values fast paths so the+/// bucket set AND the "other" count stay byte-identical across paths.+pub(crate) fn apply_terms_size_pipeline<T>(+ rows: Vec<T>,+ count_of: impl Fn(&T) -> u64,+ cap: Option<usize>,+ shard_size_param: Option<usize>,+ min_doc_count: u64,+) -> (Vec<T>, u64) {+ let mut rows = rows;+ match cap {+ Some(size) => {+ let mut sum_other: u64 = 0;+ let shard_size = shard_size_param+ .unwrap_or(size + size / 2 + 10)+ .max(size);+ if rows.len() > shard_size {+ for r in &rows[shard_size..] {+ sum_other += count_of(r);+ }+ rows.truncate(shard_size);+ }+ let mut fin: Vec<T> = Vec::with_capacity(rows.len().min(size));+ for r in rows {+ if count_of(&r) < min_doc_count {+ continue; // ES final-reduce silent drop+ }+ if fin.len() >= size {+ sum_other += count_of(&r);+ } else {+ fin.push(r);+ }+ }+ (fin, sum_other)+ }+ None => {+ rows.retain(|r| count_of(r) >= min_doc_count);+ (rows, 0)+ }+ }+}+/// ES returns typed terms-bucket keys: numbers as numbers, booleans as/// 1/0 (with key_as_string "true"/"false"), dates as epoch millis/// (with key_as_string as the ISO string). Everything else stays@@ -8749,17 +8816,51 @@ fn run_multi_terms(// Sort by doc_count descending. Partial-sort: O(M) select_nth_unstable_by// + O(N log N) prefix sort, vs O(M log M) for full sort. Wins big for- // wide multi-terms aggs over high-cardinality compound keys.+ // wide multi-terms aggs over high-cardinality compound keys. The+ // partial-select boundary is ES's shard_size (>= size), because the+ // buckets cut there only need their SUMMED doc count (for+ // sum_other_doc_count), not their order.let cmp = |a: &(Vec<String>, Vec<usize>), b: &(Vec<String>, Vec<usize>)| {b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(&b.0))};let mut sorted: Vec<(Vec<String>, Vec<usize>)> = bucket_map.into_iter().collect();- let n = size.min(sorted.len());- if n > 0 && n < sorted.len() {- sorted.select_nth_unstable_by(n, cmp);- sorted.truncate(n);+ let min_doc_count = params+ .get("min_doc_count")+ .and_then(Value::as_u64)+ .unwrap_or(1);+ let shard_size_param: Option<usize> = params+ .get("shard_size")+ .and_then(Value::as_u64)+ .map(|v| v as usize);+ // `size: 0` mirrors the terms-agg xerj extension: return all buckets.+ let cap: Option<usize> = if size == 0 { None } else { Some(size) };+ let mut sum_other_doc_count: u64 = 0;+ let mut effective_shard_size: Option<usize> = None;+ if let Some(sz) = cap {+ let shard_size = shard_size_param.unwrap_or(sz + sz / 2 + 10).max(sz);+ effective_shard_size = Some(shard_size);+ if shard_size < sorted.len() {+ sorted.select_nth_unstable_by(shard_size, cmp);+ for (_, d) in &sorted[shard_size..] {+ sum_other_doc_count += d.len() as u64;+ }+ sorted.truncate(shard_size);+ }}sorted.sort_by(cmp);+ // Final reduce: min_doc_count silent drop + size cut → other (same+ // ES single-shard accounting as run_terms). The shard cut already+ // happened above at `effective_shard_size`; passing the same value+ // makes the pipeline's internal cut a no-op (rows.len() <= shard_size)+ // so nothing is double-counted.+ let (sorted, overflow_other) = apply_terms_size_pipeline(+ sorted,+ |r| r.1.len() as u64,+ cap,+ effective_shard_size,+ min_doc_count,+ );+ sum_other_doc_count += overflow_other;let buckets: Vec<Value> = sorted.iter()@@ -8787,7 +8888,7 @@ fn run_multi_terms(json!({"doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0,+ "sum_other_doc_count": sum_other_doc_count,"buckets": buckets})}@@ -9163,9 +9264,20 @@ pub fn run_agg_fast(} else {sorted.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));}- if let Some(n) = cap {- sorted.truncate(n);- }+ // Same ES single-shard truncation accounting as `run_terms`+ // (shard-size cut → min_doc_count reduce-drop → size cut) so+ // this doc-values arm emits the identical bucket set and the+ // REAL sum_other_doc_count.+ let min_doc_count = params+ .get("min_doc_count")+ .and_then(Value::as_u64)+ .unwrap_or(1);+ let shard_size_param: Option<usize> = params+ .get("shard_size")+ .and_then(Value::as_u64)+ .map(|v| v as usize);+ let (sorted, sum_other_doc_count) =+ apply_terms_size_pipeline(sorted, |r| r.1, cap, shard_size_param, min_doc_count);let buckets: Vec<Value> = sorted.into_iter()@@ -9174,7 +9286,7 @@ pub fn run_agg_fast(FastAggResult::Value(json!({"doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0,+ "sum_other_doc_count": sum_other_doc_count,"buckets": buckets}))}@@ -9388,9 +9500,20 @@ pub fn run_agg_fast_filtered(} else {sorted.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));}- if let Some(n) = cap {- sorted.truncate(n);- }+ // Same ES single-shard truncation accounting as `run_terms`+ // (shard-size cut → min_doc_count reduce-drop → size cut) so+ // this doc-values arm emits the identical bucket set and the+ // REAL sum_other_doc_count.+ let min_doc_count = params+ .get("min_doc_count")+ .and_then(Value::as_u64)+ .unwrap_or(1);+ let shard_size_param: Option<usize> = params+ .get("shard_size")+ .and_then(Value::as_u64)+ .map(|v| v as usize);+ let (sorted, sum_other_doc_count) =+ apply_terms_size_pipeline(sorted, |r| r.1, cap, shard_size_param, min_doc_count);let buckets: Vec<Value> = sorted.into_iter()@@ -9399,7 +9522,7 @@ pub fn run_agg_fast_filtered(FastAggResult::Value(json!({"doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0,+ "sum_other_doc_count": sum_other_doc_count,"buckets": buckets}))}diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 10c6d46e9..75913aa00 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -2643,10 +2643,13 @@ impl<'a> FastCtx<'a> {// Order + truncate, mirroring run_terms' default and _count/_key// orders (cmp_terms_by_orders with no sub-agg references).- let mut candidates: Vec<(String, TermState)> = terms_map- .into_iter()- .filter(|(_, st)| st.count >= min_doc_count)- .collect();+ //+ // The min_doc_count filter is NOT applied before the sort: ES drops+ // min_doc_count failures at final reduce, AFTER the shard-size cut.+ // `apply_terms_size_pipeline` reproduces the exact ES single-shard+ // staging (shard cut → silent min drop → size cut) and returns the+ // REAL sum_other_doc_count — byte-identical to the brute run_terms.+ let mut candidates: Vec<(String, TermState)> = terms_map.into_iter().collect();candidates.sort_by(|a, b| {cmp_by_orders(&(a.0.clone(), a.1.count),@@ -2654,9 +2657,17 @@ impl<'a> FastCtx<'a> {&orders,)});- if let Some(n) = cap {- candidates.truncate(n);- }+ let shard_size_param: Option<usize> = params+ .get("shard_size")+ .and_then(Value::as_u64)+ .map(|v| v as usize);+ let (candidates, sum_other_doc_count) = crate::aggs::apply_terms_size_pipeline(+ candidates,+ |r| r.1.count,+ cap,+ shard_size_param,+ min_doc_count,+ );let mut buckets: Vec<Value> = Vec::with_capacity(candidates.len());for (key, st) in candidates {@@ -2678,7 +2689,7 @@ impl<'a> FastCtx<'a> {Some(json!({"doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0,+ "sum_other_doc_count": sum_other_doc_count,"buckets": buckets}))}