Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(aggs): the columnar histogram paths honour config.limits.max_buckets (#125)

xerj-orgea7ce28ce8f9
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index fda4eff38..bcd4f7f08 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -157,6 +157,49 @@ pub fn max_buckets() -> usize {
}
}
+/// [`max_buckets`] as an `i64`, for the gap-filling histogram loops whose
+/// bucket keys and spans are `i64`.
+///
+/// Exists so the columnar fast path (`fast_aggs.rs`) and this brute
+/// evaluator read the *same* cap through the *same* accessor: both used to
+/// carry their own `const MAX_BUCKETS: i64 = 65_536`, which silently ignored
+/// `config.limits.max_buckets` and the per-call [`AggLimits`] override.
+/// Callers hoist this above their loop, exactly like `max_buckets()`.
+///
+/// The `min` is a saturating cast, not a policy: the cap is a `usize`, and on
+/// a 64-bit target a configured cap above `i64::MAX` would otherwise wrap
+/// negative and reject every aggregation.
+#[inline]
+pub(crate) fn max_buckets_i64() -> i64 {
+ max_buckets().min(i64::MAX as usize) as i64
+}
+
+/// Diagnostic counter: aggregation requests answered by the columnar fast
+/// path in `fast_aggs.rs` instead of this brute evaluator.
+///
+/// Monotonic and process-wide, incremented with `Relaxed` once per served
+/// request. Nothing in the engine reads it to make a decision.
+///
+/// The two executors are *intended* to return byte-identical results, which is
+/// why a caller cannot tell them apart from the response and needs this signal.
+/// That intent is not yet fact: #99, #104 and #114 each closed a case where
+/// they diverged, and `exec_terms` still materialises terms without a bucket
+/// cap where the brute path applies one. Treat the invariant as the thing the
+/// tests are enforcing, not as something already true.
+static FAST_PATH_AGGS_SERVED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+
+/// Number of aggregation requests served by the columnar fast path since
+/// process start. Read two of these around a search to learn whether that
+/// search took the fast path.
+pub fn fast_path_aggs_served() -> u64 {
+ FAST_PATH_AGGS_SERVED.load(std::sync::atomic::Ordering::Relaxed)
+}
+
+/// Record that the columnar fast path answered one aggregation request.
+pub(crate) fn note_fast_path_agg() {
+ FAST_PATH_AGGS_SERVED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+}
+
/// Per-call aggregation limits. Passed to [`run_aggs_with_limits`] by callers
/// that need a limit other than the node-wide default (a per-request
/// `search.max_buckets` override, or a test pinning the cap for one call).
@@ -5904,7 +5947,9 @@ fn run_date_histogram(
.and_then(Value::as_bool)
.unwrap_or(false);
- const MAX_BUCKETS: i64 = 65_536;
+ // `search.max_buckets`, read ONCE for this aggregation (node-wide default
+ // or the per-call `AggLimits` override) — not per probed bucket below.
+ let bucket_cap = max_buckets_i64();
// Compute base range: from data, or from hard_bounds / extended_bounds.
let mut range_min = buckets.keys().min().cloned();
@@ -5984,16 +6029,16 @@ fn run_date_histogram(
let mut bucket_keys: Vec<i64> = if min_doc_count > 0 {
buckets.keys().cloned().collect()
} else if let (Some(min_key), Some(max_key)) = (range_min, range_max) {
- // Fill gaps between min and max. Enforce MAX_BUCKETS cap.
+ // Fill gaps between min and max. Enforce the bucket cap.
let mut span: i64 = 0;
let mut probe = min_key;
while probe <= max_key {
span += 1;
- if span > MAX_BUCKETS {
+ if span > bucket_cap {
return json!({
"error": format!(
"Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
- MAX_BUCKETS
+ bucket_cap
),
"__error_status__": 400u32,
});
@@ -7098,7 +7143,9 @@ fn run_histogram(
}
}
- const MAX_BUCKETS: i64 = 65_536;
+ // `search.max_buckets`, read ONCE for this aggregation — same accessor,
+ // same value the columnar `fast_aggs::exec_histogram` mirror reads.
+ let bucket_cap = max_buckets_i64();
let mut bucket_keys: Vec<i64> = if buckets.is_empty() && extended_min.is_none() {
Vec::new()
} else if min_doc_count > 0 && extended_min.is_none() {
@@ -7118,16 +7165,16 @@ fn run_histogram(
// numeric span — a few billion buckets would DoS the process).
// ES rejects aggs that exceed `search.max_buckets` (default 65536).
let span = max_key.saturating_sub(min_key);
- if span > MAX_BUCKETS {
+ if span > bucket_cap {
return json!({
"error": format!(
"Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
- MAX_BUCKETS
+ bucket_cap
),
"__error_status__": 400u32,
});
}
- let mut keys = Vec::with_capacity((span as usize).min(MAX_BUCKETS as usize) + 1);
+ let mut keys = Vec::with_capacity((span as usize).min(bucket_cap as usize) + 1);
let mut k = min_key;
while k <= max_key {
keys.push(k);
@@ -14555,6 +14602,52 @@ mod tests {
"cap outlived its call, got {after}"
);
}
+
+ /// Issue #121: the histogram gap-fill loops on BOTH executors read the
+ /// cap through `max_buckets_i64`, so it has to carry the per-call
+ /// override, not just the node-wide default. (`fast_aggs` calls the same
+ /// function; that it does is covered end-to-end by the
+ /// `agg_bucket_cap` integration suite.)
+ #[test]
+ fn max_buckets_i64_carries_the_per_call_override() {
+ let node_wide = max_buckets_i64();
+ assert_eq!(node_wide, max_buckets() as i64);
+ {
+ let _scope = BucketCapScope::new(7);
+ assert_eq!(max_buckets_i64(), 7, "override not visible to the i64 read");
+ }
+ assert_eq!(max_buckets_i64(), node_wide, "override outlived its scope");
+ }
+
+ /// A date_histogram whose gap fill exceeds a *per-call* cap must raise
+ /// too_many_buckets naming that cap — the brute half of the pair the
+ /// integration suite checks on the fast path.
+ #[test]
+ fn date_histogram_gap_fill_honours_the_per_call_cap() {
+ // Two docs, at 00:00 and 04:00 on 1970-01-01 → a 1h gap fill spans
+ // both ends inclusive, so it wants 5 buckets with doc counts
+ // 1, 0, 0, 0, 1.
+ let docs: Vec<Value> = (0..2)
+ .map(|i| json!({ "ts": i * 4 * 3_600_000i64 }))
+ .collect();
+ let agg = json!({
+ "h": { "date_histogram": { "field": "ts", "fixed_interval": "1h" } }
+ });
+
+ let capped = run_aggs_with_limits(&agg, &docs, &docs, AggLimits { max_buckets: 4 });
+ let err = capped["h"]["error"].as_str().unwrap_or_default();
+ assert!(
+ err.contains("Trying to create too many buckets") && err.contains("[4]"),
+ "expected a too_many_buckets naming the per-call cap, got {capped}"
+ );
+
+ let ok = run_aggs_with_limits(&agg, &docs, &docs, AggLimits { max_buckets: 5 });
+ assert_eq!(
+ ok["h"]["buckets"].as_array().map(Vec::len),
+ Some(5),
+ "a span exactly at the cap must still be served, got {ok}"
+ );
+ }
}
#[cfg(test)]
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 083abeabf..8348c05c2 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -695,6 +695,12 @@ impl Index {
// fast path is gated on there being no deletes, so live == physical).
let top_doc_count = filtered_total.unwrap_or(total_physical);
let result = ctx.eval_aggs_object(aggs_obj, top_doc_count)?;
+ // Served columnarly — record it. The two executors are INTENDED to
+ // return the same bytes, so a caller cannot tell them apart from the
+ // response and a test needs this signal (see
+ // `aggs::fast_path_aggs_served`, which documents why that intent is
+ // not yet fact).
+ crate::aggs::note_fast_path_agg();
Some((Value::Object(result), filtered_total))
}
}
@@ -4319,7 +4325,14 @@ impl<'a> FastCtx<'a> {
.unwrap_or(false);
// Bucket-key set: gap-fill when min_doc_count == 0 (brute default).
- const MAX_BUCKETS: i64 = 65_536;
+ //
+ // `search.max_buckets` read ONCE here, above the probe loop — the same
+ // `aggs::max_buckets_i64()` accessor `aggs::run_date_histogram` reads,
+ // so `config.limits.max_buckets` and a per-call `AggLimits` override
+ // bind this path too. It used to be a hardcoded 65 536, which meant an
+ // operator's lowered cap applied only when a query happened to miss
+ // the columnar path.
+ let bucket_cap = crate::aggs::max_buckets_i64();
let mut final_keys: Vec<i64> = if min_doc_count > 0 {
bucket_keys.clone()
} else if let (Some(&min_key), Some(&max_key)) =
@@ -4329,11 +4342,11 @@ impl<'a> FastCtx<'a> {
let mut probe = min_key;
while probe <= max_key {
span += 1;
- if span > MAX_BUCKETS {
+ if span > bucket_cap {
return Some(json!({
"error": format!(
"Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
- MAX_BUCKETS
+ bucket_cap
),
"__error_status__": 400u32,
}));
@@ -4572,7 +4585,9 @@ impl<'a> FastCtx<'a> {
}
// ── Bucket-key set (mirrors run_histogram exactly) ────────────────
- const MAX_BUCKETS: i64 = 65_536;
+ // Including the cap: same `aggs::max_buckets_i64()` accessor, read
+ // once above the fill, so the two paths reject the same spans.
+ let bucket_cap = crate::aggs::max_buckets_i64();
let mut final_keys: Vec<i64> = if bucket_ids.is_empty() && extended_min.is_none() {
Vec::new()
} else if min_doc_count > 0 && extended_min.is_none() {
@@ -4583,16 +4598,16 @@ impl<'a> FastCtx<'a> {
let min_key = extended_min.map(key_of).or(data_min).unwrap_or(0);
let max_key = extended_max.map(key_of).or(data_max).unwrap_or(0);
let span = max_key.saturating_sub(min_key);
- if span > MAX_BUCKETS {
+ if span > bucket_cap {
return Some(json!({
"error": format!(
"Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
- MAX_BUCKETS
+ bucket_cap
),
"__error_status__": 400u32,
}));
}
- let mut keys = Vec::with_capacity((span as usize).min(MAX_BUCKETS as usize) + 1);
+ let mut keys = Vec::with_capacity((span as usize).min(bucket_cap as usize) + 1);
let mut k = min_key;
while k <= max_key {
keys.push(k);
diff --git a/engine/crates/xerj-engine/tests/agg_bucket_cap.rs b/engine/crates/xerj-engine/tests/agg_bucket_cap.rs
new file mode 100644
index 000000000..298bb42eb
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/agg_bucket_cap.rs
@@ -0,0 +1,211 @@
+//! `config.limits.max_buckets` must bind the columnar fast path exactly as it
+//! binds the brute path (issue #121).
+//!
+//! `fast_aggs.rs` used to carry its own `const MAX_BUCKETS: i64 = 65_536` in
+//! `exec_date_histogram` and `exec_histogram`, so an operator who lowered the
+//! cap to protect memory got that protection only on queries that happened to
+//! miss the columnar path — and the discrepancy was invisible, because the two
+//! executors otherwise return byte-identical results.
+//!
+//! Which executor answered is read from `aggs::fast_path_aggs_served`; the
+//! brute half of each pair is forced by keeping that index under the columnar
+//! path's own size gate (`fast_aggs::FAST_AGG_MIN_DOCS`, 10 000 docs) — same
+//! agg, same bucket span, different executor.
+//!
+//! This suite gets its own test binary on purpose: `Engine::new` installs the
+//! cap in a process-wide static (`aggs::set_max_buckets`), so every engine in
+//! the file must agree on one value and no other suite may observe it. For the
+//! same reason it is deliberately ONE test function — parallel tests in one
+//! binary would race on both the cap and the fast-path counter.
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+use xerj_engine::aggs::fast_path_aggs_served;
+use xerj_engine::bulk::process_bulk;
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+/// Lowered far below the stock 65 536 so a corpus that trips it is still
+/// small enough to index inside a test.
+const CAP: usize = 64;
+
+/// Docs in an index that must take the columnar path — `fast_aggs` refuses to
+/// serve an index below its own `FAST_AGG_MIN_DOCS` (10 000).
+const BIG_DOCS: usize = 10_050;
+
+/// Docs in an index that must NOT take the columnar path: same shape, same
+/// bucket spans, on the other side of that gate.
+const SMALL_DOCS: usize = 900;
+
+const HOUR_MS: i64 = 3_600_000;
+/// Hour-aligned epoch-ms, so one distinct `hour` offset == one `1h` bucket.
+const BASE_MS: i64 = 1_699_999_200_000;
+
+/// A corpus shape: how many distinct `ts` hours and how many distinct `n`
+/// values its docs cycle through — i.e. how many buckets a gap-filled
+/// histogram over the whole corpus has to materialise.
+#[derive(Clone, Copy)]
+struct Shape {
+ hours: usize,
+ nvals: usize,
+}
+
+/// Exactly at the cap. `date_histogram` counts buckets, `histogram` compares
+/// `max_key - min_key` (a difference, not a count — pre-existing, and the
+/// same on both paths), so "at the cap" is `CAP` hours and `CAP + 1` values.
+const AT_CAP: Shape = Shape {
+ hours: CAP,
+ nvals: CAP + 1,
+};
+/// One bucket past the cap on both aggs.
+const OVER_CAP: Shape = Shape {
+ hours: CAP + 1,
+ nvals: CAP + 2,
+};
+
+fn ndjson_batch(index: &str, shape: Shape, ids: std::ops::Range<usize>) -> String {
+ let mut out = String::with_capacity(ids.len() * 96);
+ for i in ids {
+ out.push_str(&format!(
+ "{{\"index\":{{\"_index\":\"{index}\",\"_id\":\"{i}\"}}}}\n"
+ ));
+ let ts = BASE_MS + (i % shape.hours) as i64 * HOUR_MS;
+ let n = (i % shape.nvals) as i64;
+ out.push_str(&format!("{{\"ts\":{ts},\"n\":{n}}}\n"));
+ }
+ out
+}
+
+async fn build_index(engine: &Engine, name: &str, docs: usize, shape: Shape) {
+ engine.create_index(name, Schema::empty()).unwrap();
+ for start in (0..docs).step_by(1_000) {
+ let end = (start + 1_000).min(docs);
+ let body = ndjson_batch(name, shape, start..end);
+ let res = process_bulk(engine, Some(name), &body).await;
+ assert!(!res.errors, "bulk errors while seeding {name}");
+ }
+ // Flush to segments: the columnar path reads `.dv` sidecars, and
+ // `exec_histogram` only runs when the field has a real numeric column.
+ engine.get_index(name).unwrap().refresh().await.unwrap();
+}
+
+/// Run one `size:0 + match_all` agg request and report both the agg body and
+/// whether the columnar fast path served it.
+async fn run_agg(engine: &Engine, index: &str, agg: Value) -> (Value, u64) {
+ let idx = engine.get_index(index).unwrap();
+ let req = parse_request(&json!({
+ "query": { "match_all": {} },
+ "size": 0,
+ "aggs": { "a": agg },
+ }))
+ .unwrap();
+ let before = fast_path_aggs_served();
+ let res = idx.search(&req).await.unwrap();
+ let served = fast_path_aggs_served() - before;
+ let aggs = res.aggs.clone().expect("aggs present");
+ (aggs["a"].clone(), served)
+}
+
+/// The too_many_buckets marker the agg runners embed, if present. It carries
+/// the cap that was actually enforced — `[64]` here, `[65536]` before the fix.
+fn too_many_buckets(agg: &Value) -> Option<String> {
+ let msg = agg.get("error")?.as_str()?;
+ assert_eq!(agg.get("__error_status__"), Some(&json!(400)));
+ assert!(
+ msg.contains("Trying to create too many buckets"),
+ "unexpected agg error: {msg}"
+ );
+ Some(msg.to_string())
+}
+
+fn bucket_count(agg: &Value) -> usize {
+ agg["buckets"]
+ .as_array()
+ .unwrap_or_else(|| panic!("expected buckets, got {agg}"))
+ .len()
+}
+
+#[tokio::test]
+async fn lowered_max_buckets_binds_the_columnar_fast_path_and_the_brute_path_alike() {
+ let dir = TempDir::new().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ config.limits.max_buckets = CAP;
+ let engine = Engine::new(config).expect("engine::new");
+
+ build_index(&engine, "big-over", BIG_DOCS, OVER_CAP).await;
+ build_index(&engine, "big-at", BIG_DOCS, AT_CAP).await;
+ build_index(&engine, "small-over", SMALL_DOCS, OVER_CAP).await;
+ build_index(&engine, "small-at", SMALL_DOCS, AT_CAP).await;
+
+ let date_hist = json!({"date_histogram": {"field": "ts", "fixed_interval": "1h"}});
+ let num_hist = json!({"histogram": {"field": "n", "interval": 1}});
+
+ // ── Fast path: the lowered cap must trip it ──────────────────────────
+ let (agg, served) = run_agg(&engine, "big-over", date_hist.clone()).await;
+ assert_eq!(served, 1, "date_histogram did not take the columnar path");
+ let fast_date_err = too_many_buckets(&agg).unwrap_or_else(|| {
+ panic!(
+ "fast path ignored max_buckets={CAP} and returned {} buckets: {agg}",
+ bucket_count(&agg)
+ )
+ });
+ assert!(
+ fast_date_err.contains(&format!("[{CAP}]")),
+ "fast path reported a cap it did not enforce: {fast_date_err}"
+ );
+
+ let (agg, served) = run_agg(&engine, "big-over", num_hist.clone()).await;
+ assert_eq!(served, 1, "histogram did not take the columnar path");
+ let fast_num_err = too_many_buckets(&agg).unwrap_or_else(|| {
+ panic!(
+ "fast path ignored max_buckets={CAP} and returned {} buckets: {agg}",
+ bucket_count(&agg)
+ )
+ });
+ assert!(
+ fast_num_err.contains(&format!("[{CAP}]")),
+ "fast path reported a cap it did not enforce: {fast_num_err}"
+ );
+
+ // ── Fast path: a span exactly at the cap must still be answered ──────
+ let (agg, served) = run_agg(&engine, "big-at", date_hist.clone()).await;
+ assert_eq!(served, 1, "date_histogram did not take the columnar path");
+ assert_eq!(bucket_count(&agg), CAP, "cap-sized span must be served");
+
+ let (agg, served) = run_agg(&engine, "big-at", num_hist.clone()).await;
+ assert_eq!(served, 1, "histogram did not take the columnar path");
+ assert_eq!(bucket_count(&agg), CAP + 1, "cap-sized span must be served");
+
+ // ── Brute path: same shapes, below the columnar path's size gate ─────
+ // Same trip point, same message, same bucket counts.
+ let (agg, served) = run_agg(&engine, "small-over", date_hist.clone()).await;
+ assert_eq!(served, 0, "the small index must not take the fast path");
+ assert_eq!(
+ too_many_buckets(&agg),
+ Some(fast_date_err),
+ "brute and fast date_histogram disagree about the cap"
+ );
+
+ let (agg, served) = run_agg(&engine, "small-over", num_hist.clone()).await;
+ assert_eq!(served, 0, "the small index must not take the fast path");
+ assert_eq!(
+ too_many_buckets(&agg),
+ Some(fast_num_err),
+ "brute and fast histogram disagree about the cap"
+ );
+
+ let (agg, served) = run_agg(&engine, "small-at", date_hist).await;
+ assert_eq!(served, 0, "the small index must not take the fast path");
+ assert_eq!(bucket_count(&agg), CAP, "brute dropped a cap-sized span");
+
+ let (agg, served) = run_agg(&engine, "small-at", num_hist).await;
+ assert_eq!(served, 0, "the small index must not take the fast path");
+ assert_eq!(
+ bucket_count(&agg),
+ CAP + 1,
+ "brute dropped a cap-sized span"
+ );
+}