Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(aggs): keep histogram data buckets outside extended_bounds (#1819) (#1823)

Amartya Vikram Singh8de3194e47ea
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index b8125f489..1ea3a5ce5 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -7880,14 +7880,20 @@ fn run_histogram(
} else {
let data_min = buckets.keys().min().copied();
let data_max = buckets.keys().max().copied();
- let min_key = extended_min
- .map(|v| ((v - offset) / interval).floor() as i64)
- .or(data_min)
- .unwrap_or(0);
- let max_key = extended_max
- .map(|v| ((v - offset) / interval).floor() as i64)
- .or(data_max)
- .unwrap_or(0);
+ // #1819: `extended_bounds` only ADDS empty buckets — the fill range
+ // is the wider of the bounds and the data, so a bound inside the data
+ // never drops a bucket that holds documents (only `hard_bounds`, below,
+ // does). `date_histogram` widens the same way (`range_min.min(key)`).
+ let ext_min_key = extended_min.map(|v| ((v - offset) / interval).floor() as i64);
+ let ext_max_key = extended_max.map(|v| ((v - offset) / interval).floor() as i64);
+ let min_key = match (ext_min_key, data_min) {
+ (Some(e), Some(d)) => e.min(d),
+ (e, d) => e.or(d).unwrap_or(0),
+ };
+ let max_key = match (ext_max_key, data_max) {
+ (Some(e), Some(d)) => e.max(d),
+ (e, d) => e.or(d).unwrap_or(0),
+ };
// Guard against pathological inputs (tiny intervals with a wide
// numeric span — a few billion buckets would DoS the process).
// ES rejects aggs that exceed `search.max_buckets` (default 65536).
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index cb9e78797..a496da6cb 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -5021,8 +5021,16 @@ impl<'a> FastCtx<'a> {
} else {
let data_min = bucket_ids.keys().min().copied();
let data_max = bucket_ids.keys().max().copied();
- let min_key = extended_min.map(key_of).or(data_min).unwrap_or(0);
- let max_key = extended_max.map(key_of).or(data_max).unwrap_or(0);
+ // #1819: the wider of the bounds and the data, as in
+ // `run_histogram` — `extended_bounds` never drops data buckets.
+ let min_key = match (extended_min.map(key_of), data_min) {
+ (Some(e), Some(d)) => e.min(d),
+ (e, d) => e.or(d).unwrap_or(0),
+ };
+ let max_key = match (extended_max.map(key_of), data_max) {
+ (Some(e), Some(d)) => e.max(d),
+ (e, d) => e.or(d).unwrap_or(0),
+ };
let span = max_key.saturating_sub(min_key);
if span > bucket_cap {
return Some(json!({
diff --git a/engine/crates/xerj-engine/tests/histogram_extended_bounds.rs b/engine/crates/xerj-engine/tests/histogram_extended_bounds.rs
new file mode 100644
index 000000000..18214c0cf
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/histogram_extended_bounds.rs
@@ -0,0 +1,159 @@
+//! Issue #1819: `histogram` `extended_bounds` must only ADD empty buckets.
+//! Buckets holding data outside the bounds stay, as in Elasticsearch 8.13.4;
+//! only `hard_bounds` drops them.
+//!
+//! ROOT CAUSE: both executors picked the gap-fill range as
+//! `extended_min.or(data_min)` / `extended_max.or(data_max)` — the bound
+//! *instead of* the wider of the bound and the data — so a bound inside the
+//! data range clipped the buckets on either side of it, the same as
+//! `hard_bounds`. The brute runner (`aggs::run_histogram`) and its columnar
+//! mirror (`fast_aggs::exec_histogram`) carried the same two lines.
+//! `date_histogram` already widened (`range_min.min(key)` /
+//! `range_max.max(key)`), which is why it matched ES.
+//!
+//! Both executors are pinned: the columnar path only serves an index at or
+//! above `fast_aggs::FAST_AGG_MIN_DOCS` (10 000 docs), so the same values are
+//! indexed once below that gate and once above it, and
+//! `aggs::fast_path_aggs_served` says which executor answered. That counter
+//! is process-global, so this is deliberately ONE test function.
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+use xerj_engine::aggs::fast_path_aggs_served;
+use xerj_engine::bulk::process_bulk;
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+/// One case: the histogram agg, the expected (key, doc_count) buckets, and a label.
+type HistogramCase = (Value, Vec<(i64, u64)>, &'static str);
+
+/// The #1819 reproduction's values. ES 8.13.4, `interval: 5`,
+/// `extended_bounds: {min: 0, max: 10}`:
+/// `-5:1, 0:2, 5:0, 10:1, 15:0, 20:0, 25:1`.
+const VALUES: [f64; 5] = [-2.0, 1.0, 3.0, 12.0, 27.0];
+
+/// Below the columnar gate: one copy of the values.
+const SMALL_DOCS: usize = VALUES.len();
+/// Above the columnar gate: 2 010 copies of the values (10 050 docs), so
+/// every expected count is the ES count times `BIG_REPEAT`.
+const BIG_REPEAT: usize = 2_010;
+
+async fn build_index(engine: &Engine, name: &str, docs: usize) {
+ engine.create_index(name, Schema::empty()).unwrap();
+ for start in (0..docs).step_by(1_000) {
+ let end = (start + 1_000).min(docs);
+ let mut body = String::new();
+ for i in start..end {
+ body.push_str(&format!(
+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{i}\"}}}}\n"
+ ));
+ body.push_str(&format!("{{\"v\":{}}}\n", VALUES[i % VALUES.len()]));
+ }
+ let res = process_bulk(engine, Some(name), &body).await;
+ assert!(!res.errors, "bulk errors while seeding {name}");
+ }
+ // The columnar path reads segment `.dv` sidecars; flush both indices so
+ // the only difference between them is the size gate.
+ engine.get_index(name).unwrap().refresh().await.unwrap();
+}
+
+/// `(key, doc_count)` pairs of one `size:0` histogram, and whether the
+/// columnar fast path served it.
+async fn histogram(engine: &Engine, index: &str, agg: Value) -> (Vec<(i64, u64)>, u64) {
+ let idx = engine.get_index(index).unwrap();
+ let req = parse_request(&json!({
+ "query": {"match_all": {}},
+ "size": 0,
+ "aggs": {"h": agg},
+ }))
+ .unwrap();
+ let before = fast_path_aggs_served();
+ let res = idx.search(&req).await.unwrap();
+ let served = fast_path_aggs_served() - before;
+ let aggs = res.aggs.clone().expect("aggs present");
+ let buckets = aggs["h"]["buckets"]
+ .as_array()
+ .unwrap_or_else(|| panic!("expected buckets, got {aggs}"))
+ .iter()
+ .map(|b| {
+ (
+ b["key"].as_f64().expect("numeric key") as i64,
+ b["doc_count"].as_u64().expect("doc_count"),
+ )
+ })
+ .collect();
+ (buckets, served)
+}
+
+fn scaled(expected: &[(i64, u64)], by: usize) -> Vec<(i64, u64)> {
+ expected.iter().map(|&(k, n)| (k, n * by as u64)).collect()
+}
+
+#[tokio::test]
+async fn extended_bounds_keeps_data_buckets_outside_the_bounds_on_both_executors() {
+ let dir = TempDir::new().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ let engine = Engine::new(config).expect("engine::new");
+
+ build_index(&engine, "small", SMALL_DOCS).await;
+ build_index(&engine, "big", VALUES.len() * BIG_REPEAT).await;
+
+ // (agg, expected buckets for ONE copy of VALUES, what the case pins)
+ let cases: Vec<HistogramCase> = vec![
+ (
+ json!({"histogram": {"field": "v", "interval": 5,
+ "extended_bounds": {"min": 0, "max": 10}}}),
+ vec![(-5, 1), (0, 2), (5, 0), (10, 1), (15, 0), (20, 0), (25, 1)],
+ "#1819: bounds inside the data keep the data on both sides",
+ ),
+ (
+ json!({"histogram": {"field": "v", "interval": 5,
+ "extended_bounds": {"min": -20, "max": 40}}}),
+ vec![
+ (-20, 0),
+ (-15, 0),
+ (-10, 0),
+ (-5, 1),
+ (0, 2),
+ (5, 0),
+ (10, 1),
+ (15, 0),
+ (20, 0),
+ (25, 1),
+ (30, 0),
+ (35, 0),
+ (40, 0),
+ ],
+ "bounds wider than the data still add empty buckets",
+ ),
+ (
+ json!({"histogram": {"field": "v", "interval": 5,
+ "extended_bounds": {"min": 10}}}),
+ vec![(-5, 1), (0, 2), (5, 0), (10, 1), (15, 0), (20, 0), (25, 1)],
+ "a min-only bound above the data min drops nothing",
+ ),
+ (
+ json!({"histogram": {"field": "v", "interval": 5,
+ "hard_bounds": {"min": 0, "max": 10}}}),
+ vec![(0, 2), (5, 0), (10, 1)],
+ "hard_bounds still drops buckets outside it (unchanged)",
+ ),
+ ];
+
+ for (agg, expected, what) in cases {
+ let (got, served) = histogram(&engine, "small", agg.clone()).await;
+ assert_eq!(served, 0, "the small index must take the brute path");
+ assert_eq!(got, expected, "brute path: {what}: {agg}");
+
+ let (got, served) = histogram(&engine, "big", agg.clone()).await;
+ assert_eq!(served, 1, "the big index must take the columnar path");
+ assert_eq!(
+ got,
+ scaled(&expected, BIG_REPEAT),
+ "columnar path: {what}: {agg}"
+ );
+ }
+}
diff --git a/engine/tests/es-compat-yaml/yaml/search/772_histogram_extended_bounds_inside_data.yml b/engine/tests/es-compat-yaml/yaml/search/772_histogram_extended_bounds_inside_data.yml
new file mode 100644
index 000000000..830c61142
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/772_histogram_extended_bounds_inside_data.yml
@@ -0,0 +1,66 @@
+# #1819: `extended_bounds` only ADDS empty buckets. Buckets that hold data
+# outside the bounds stay; only `hard_bounds` drops them. Expected keys and
+# counts are Elasticsearch 8.13.4's for the requests in #1819.
+setup:
+ - do:
+ indices.create:
+ index: hist_eb
+ body:
+ mappings:
+ properties:
+ v: { type: double }
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - '{"index": {"_index": "hist_eb", "_id": "1"}}'
+ - '{"v": -2}'
+ - '{"index": {"_index": "hist_eb", "_id": "2"}}'
+ - '{"v": 1}'
+ - '{"index": {"_index": "hist_eb", "_id": "3"}}'
+ - '{"v": 3}'
+ - '{"index": {"_index": "hist_eb", "_id": "4"}}'
+ - '{"v": 12}'
+ - '{"index": {"_index": "hist_eb", "_id": "5"}}'
+ - '{"v": 27}'
+
+---
+"extended_bounds inside the data keeps the data buckets on both sides":
+ - do:
+ search:
+ index: hist_eb
+ body:
+ size: 0
+ aggs:
+ h:
+ histogram:
+ field: v
+ interval: 5
+ extended_bounds: { min: 0, max: 10 }
+ - length: { aggregations.h.buckets: 7 }
+ - match: { aggregations.h.buckets.0.key: -5.0 }
+ - match: { aggregations.h.buckets.0.doc_count: 1 }
+ - match: { aggregations.h.buckets.1.key: 0.0 }
+ - match: { aggregations.h.buckets.1.doc_count: 2 }
+ - match: { aggregations.h.buckets.2.doc_count: 0 }
+ - match: { aggregations.h.buckets.3.key: 10.0 }
+ - match: { aggregations.h.buckets.3.doc_count: 1 }
+ - match: { aggregations.h.buckets.6.key: 25.0 }
+ - match: { aggregations.h.buckets.6.doc_count: 1 }
+
+---
+"hard_bounds still drops the buckets outside it":
+ - do:
+ search:
+ index: hist_eb
+ body:
+ size: 0
+ aggs:
+ h:
+ histogram:
+ field: v
+ interval: 5
+ hard_bounds: { min: 0, max: 10 }
+ - length: { aggregations.h.buckets: 3 }
+ - match: { aggregations.h.buckets.0.key: 0.0 }
+ - match: { aggregations.h.buckets.2.key: 10.0 }