AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(aggs): metric aggs fold ALL values of a multi-valued field (#842) (#843)
xerj-org5831d19f400a
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 9457e1e4b..f60fd2301 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -121,8 +121,8 @@ pub const SUPPORTED_AGG_TYPES: &[&str] = &[/// their own cache scoped to the (filtered) bucket docs.struct FieldCache<'d> {docs: &'d [Value],- /// field → Vec<Option<f64>>- numeric: HashMap<String, Vec<Option<f64>>>,+ /// field → Vec<Vec<f64>> (ALL numeric values per doc, for metric aggs)+ numeric_multi: HashMap<String, Vec<Vec<f64>>>,/// field → Vec<Vec<String>>strings: HashMap<String, Vec<Vec<String>>>,}@@ -131,18 +131,23 @@ impl<'d> FieldCache<'d> {fn new(docs: &'d [Value]) -> Self {Self {docs,- numeric: HashMap::new(),+ numeric_multi: HashMap::new(),strings: HashMap::new(),}}- fn get_numeric(&mut self, field: &str) -> &[Option<f64>] {- self.numeric.entry(field.to_string()).or_insert_with(|| {- self.docs- .iter()- .map(|d| extract_numeric(d, field))- .collect()- })+ /// Every numeric value per doc (a multi-valued field yields all its+ /// values, ES semantics — not just the first). Coercion matches+ /// `extract_numeric` (number, numeric string, or date string → epoch ms).+ fn get_numeric_multi(&mut self, field: &str) -> &[Vec<f64>] {+ self.numeric_multi+ .entry(field.to_string())+ .or_insert_with(|| {+ self.docs+ .iter()+ .map(|d| extract_metric_numeric_values(d, field))+ .collect()+ })}fn get_strings(&mut self, field: &str) -> &[Vec<String>] {@@ -3292,6 +3297,59 @@ pub(crate) fn extract_numeric_values(doc: &Value, field: &str) -> Vec<f64> {out}+/// All numeric values of a field for a METRIC agg (avg/sum/min/max/stats/…),+/// matching `extract_numeric`'s per-value coercion — a plain number, a numeric+/// string, or a date string parsed to epoch ms — but across EVERY value of a+/// multi-valued field, not just the first. `extract_numeric_values` above is+/// NOT used here because it lacks the date-string fallback that date metric+/// aggs depend on.+pub(crate) fn extract_metric_numeric_values(doc: &Value, field: &str) -> Vec<f64> {+ fn num_of(v: &Value) -> Option<f64> {+ match v {+ Value::Number(n) => n.as_f64(),+ Value::String(s) => s+ .parse::<f64>()+ .ok()+ .or_else(|| parse_date_ms(&Value::String(s.clone())).map(|ms| ms as f64)),+ _ => None,+ }+ }+ fn walk(v: &Value, out: &mut Vec<f64>) {+ match v {+ Value::Array(arr) => {+ for e in arr {+ walk(e, out);+ }+ }+ other => {+ if let Some(f) = num_of(other) {+ out.push(f);+ }+ }+ }+ }+ let mut out = Vec::new();+ walk(get_nested_field(doc, field), &mut out);+ out+}++/// Flatten every numeric value across all docs for a metric agg: a multi-valued+/// field contributes ALL its values (ES semantics); a doc with NO numeric value+/// contributes the `missing` placeholder once, when set.+fn metric_numeric_values(cache: &mut FieldCache, field: &str, missing: Option<f64>) -> Vec<f64> {+ let mut out = Vec::new();+ for vals in cache.get_numeric_multi(field) {+ if vals.is_empty() {+ if let Some(m) = missing {+ out.push(m);+ }+ } else {+ out.extend_from_slice(vals);+ }+ }+ out+}+/// Extract ALL parseable date values (as epoch ms) from a field.pub(crate) fn extract_date_ms_values(doc: &Value, field: &str) -> Vec<i64> {fn walk(v: &Value, out: &mut Vec<i64>) {@@ -6384,13 +6442,9 @@ fn run_avg<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->None => return json!({"value": Value::Null, "__xy_count__": 0, "__xy_sum__": 0.0}),};let missing = get_missing_value(params);- let (sum, count) = cache- .get_numeric(field)+ let (sum, count) = metric_numeric_values(cache, field, missing).iter()- .fold((0.0f64, 0usize), |(s, n), v| match v.or(missing) {- Some(x) => (s + x, n + 1),- None => (s, n),- });+ .fold((0.0f64, 0usize), |(s, n), x| (s + x, n + 1));if count == 0 {return json!({ "value": Value::Null, "__xy_count__": 0, "__xy_sum__": 0.0 });}@@ -6408,11 +6462,7 @@ fn run_sum<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->None => return json!({"value": 0.0, "__xy_agg__": "sum"}),};let missing = get_missing_value(params);- let values: Vec<f64> = cache- .get_numeric(field)- .iter()- .filter_map(|v| v.or(missing))- .collect();+ let values: Vec<f64> = metric_numeric_values(cache, field, missing);// ES semantics: a top-level `sum` with no matching docs returns 0.0// (the additive identity), but a `sum` that participated in a// bucketing agg where the bucket had zero docs is emitted as 0 in the@@ -6432,10 +6482,9 @@ fn run_min<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->None => return json!({"value": Value::Null, "__xy_agg__": "min"}),};let missing = get_missing_value(params);- let min = cache- .get_numeric(field)+ let min = metric_numeric_values(cache, field, missing).iter()- .filter_map(|v| v.or(missing))+ .copied().fold(f64::INFINITY, f64::min);if min.is_infinite() {return json!({ "value": Value::Null, "__xy_agg__": "min" });@@ -6455,10 +6504,9 @@ fn run_max<'d>(params: &Value, docs: &'d [Value], cache: &mut FieldCache<'d>) ->None => return json!({"value": Value::Null, "__xy_agg__": "max"}),};let missing = get_missing_value(params);- let max = cache- .get_numeric(field)+ let max = metric_numeric_values(cache, field, missing).iter()- .filter_map(|v| v.or(missing))+ .copied().fold(f64::NEG_INFINITY, f64::max);if max.is_infinite() {return json!({ "value": Value::Null, "__xy_agg__": "max" });@@ -6503,10 +6551,9 @@ fn run_stats<'d>(params: &Value, _docs: &'d [Value], cache: &mut FieldCache<'d>)};let missing = get_missing_value(params);- let (count, sum, min, max) = cache- .get_numeric(field)+ let (count, sum, min, max) = metric_numeric_values(cache, field, missing).iter()- .filter_map(|v| v.or(missing))+ .copied().fold((0usize, 0.0f64, f64::INFINITY, f64::NEG_INFINITY),|(cnt, s, mn, mx), x| (cnt + 1, s + x, mn.min(x), mx.max(x)),@@ -11458,11 +11505,7 @@ fn run_extended_stats<'d>(params: &Value, _docs: &'d [Value], cache: &mut FieldC// `missing: N` — docs without a numeric value for the field contribute// N instead of being skipped. Applied before count/sum/sum_of_squares.let missing_default = get_missing_value(params);- let values: Vec<f64> = cache- .get_numeric(field)- .iter()- .filter_map(|v| v.or(missing_default))- .collect();+ let values: Vec<f64> = metric_numeric_values(cache, field, missing_default);let count = values.len();if count == 0 {@@ -11643,11 +11686,7 @@ fn run_median_absolute_deviation<'d>(};let missing = get_missing_value(params);- let mut values: Vec<f64> = cache- .get_numeric(field)- .iter()- .filter_map(|v| v.or(missing))- .collect();+ let mut values: Vec<f64> = metric_numeric_values(cache, field, missing);if values.is_empty() {return json!({"value": Value::Null});}@@ -13734,6 +13773,51 @@ mod tests {use super::*;use serde_json::json;+ #[test]+ fn multivalued_numeric_metric_aggs_use_all_values() {+ // ES aggregates over EVERY value of a multi-valued field; the old path+ // used only the first (extract_numeric), so sum/avg/stats undercounted+ // and min/max could miss the extreme. `n` values: [1,20] + [3].+ let docs = vec![json!({ "n": [1, 20] }), json!({ "n": [3] })];+ let r = run_aggs(+ &json!({+ "s": {"sum": {"field": "n"}},+ "a": {"avg": {"field": "n"}},+ "mn": {"min": {"field": "n"}},+ "mx": {"max": {"field": "n"}},+ "st": {"stats": {"field": "n"}},+ "es": {"extended_stats": {"field": "n"}},+ }),+ &docs,+ );+ assert_eq!(r["s"]["value"].as_f64(), Some(24.0), "sum");+ assert_eq!(r["a"]["value"].as_f64(), Some(8.0), "avg");+ assert_eq!(r["mn"]["value"].as_f64(), Some(1.0), "min");+ assert_eq!(r["mx"]["value"].as_f64(), Some(20.0), "max");+ assert_eq!(r["st"]["count"].as_u64(), Some(3), "stats.count");+ assert_eq!(r["st"]["sum"].as_f64(), Some(24.0), "stats.sum");+ assert_eq!(r["st"]["max"].as_f64(), Some(20.0), "stats.max");+ assert_eq!(r["es"]["count"].as_u64(), Some(3), "extended_stats.count");++ // `missing`: a doc with no value contributes the placeholder once.+ let dm = vec![json!({ "n": [2, 4] }), json!({ "x": 1 })];+ let rm = run_aggs(&json!({ "s": {"sum": {"field": "n", "missing": 10}} }), &dm);+ assert_eq!(rm["s"]["value"].as_f64(), Some(16.0), "sum with missing");++ // Date-string field (single value) must still resolve via date coercion+ // — the all-values extractor keeps `extract_numeric`'s date fallback.+ let dd = vec![json!({ "d": "2016-05-03" }), json!({ "d": "2016-05-01" })];+ let rd = run_aggs(&json!({ "mn": {"min": {"field": "d"}} }), &dd);+ assert!(+ rd["mn"]["value"].as_f64().is_some(),+ "date min resolves: {rd:?}"+ );+ assert!(+ rd["mn"]["value_as_string"].is_string(),+ "date min_as_string: {rd:?}"+ );+ }+#[test]fn passage_metadata_is_not_an_aggregation_field() {let doc = json!({diff --git a/engine/crates/xerj-engine/tests/fast_aggs_mixed_segment_suppression.rs b/engine/crates/xerj-engine/tests/fast_aggs_mixed_segment_suppression.rsindex 9e35c7cb7..731e61b2a 100644--- a/engine/crates/xerj-engine/tests/fast_aggs_mixed_segment_suppression.rs+++ b/engine/crates/xerj-engine/tests/fast_aggs_mixed_segment_suppression.rs@@ -221,16 +221,15 @@ async fn fast_and_brute_agree_on_mixed_segment_numeric_metrics() {metrics over a field whose column is suppressed in ONE segment only",).await;- // The reference is the BRUTE path, asserted as it actually behaves, not- // as ES would: its sum/avg resolvers fold ONE value per doc (the array's- // first element), while value_count counts every element. So sum is- // 11 000 × 5 + 1 000 × 1, avg divides by the 12 000 docs, and- // value_count sees all 13 000 values. What this test pins is the- // fast/brute AGREEMENT above; brute's own array-metric semantics are a- // separate (pre-existing) question.- assert_eq!(brute["aggs"]["s"]["value"], 56_000.0);+ // The reference is the BRUTE path. Metric aggs now fold EVERY value of a+ // multi-valued field (ES semantics — #839-family fix), so sum is+ // 11 000 × 5 + 1 000 × (1 + 2) = 58 000, value_count sees all 13 000+ // values, and avg divides the sum by the 13 000 values (not the 12 000+ // docs). What this test pins is the fast/brute AGREEMENT above; both paths+ // resolve `score` through brute here (its column is suppressed in seg 2).+ assert_eq!(brute["aggs"]["s"]["value"], 58_000.0);assert_eq!(brute["aggs"]["n"]["value"], 13_000);- assert_eq!(brute["aggs"]["a"]["value"], 56_000.0 / 12_000.0);+ assert_eq!(brute["aggs"]["a"]["value"], 58_000.0 / 13_000.0);}/// The same metric as a SUB-agg under a fully-covered `terms` parent:@@ -260,9 +259,9 @@ async fn fast_and_brute_agree_on_mixed_segment_sub_metric() {let buckets = brute["aggs"]["by_group"]["buckets"].as_array().unwrap();assert_eq!(buckets.len(), 2);for b in buckets {- // Each group: 5 500 scalar docs × 5 + 500 array docs × 1 (brute's- // sum folds one value per doc — see the metrics test above).+ // Each group: 5 500 scalar docs × 5 + 500 array docs × (1 + 2) — the+ // sub-metric sum folds every value of the multi-valued field (ES).assert_eq!(b["doc_count"], 6_000);- assert_eq!(b["s"]["value"], 28_000.0);+ assert_eq!(b["s"]["value"], 29_000.0);}}