Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): bound aggregation corpus memory under max_query_memory_mb (#464)

xerj-orgbb1e8dd7fd67
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 66a6a9fa7..df4308674 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -5132,7 +5132,7 @@ fn compile_top_pred(filter: &Value) -> Option<Pred> {
/// costs an extra clone, never correctness — so this deliberately mirrors
/// the substring semantics (`top_hits` anywhere; the meta fields as exact
/// key/string tokens, i.e. the serialized quoted form).
-pub(super) fn agg_tree_mentions_meta(v: &Value) -> bool {
+fn agg_tree_mentions_meta(v: &Value) -> bool {
fn is_meta(s: &str) -> bool {
s.contains("top_hits") || s == "_id" || s == "_index" || s == "_seq_no"
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 563359e5c..e7ab1ddf9 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -13863,33 +13863,6 @@ impl Index {
}
}
- // #464: the guard above only covers the search RESULT WINDOW. An
- // aggregation runs with `size: 0`, so that estimate is 0 — yet a
- // meta-observing agg (`top_hits`, or one targeting `_id`/`_index`/
- // `_seq_no`) deep-clones EVERY buffered memtable document to build its
- // owned-source view (`fast_aggs::MemDocs::Owned`), which under a bulk
- // writer is 10^4-10^5 full-source clones. That allocation escaped
- // `max_query_memory_mb` entirely. Estimate it here — before either the
- // fast or the brute agg path runs — and reject up-front. Bounds the
- // owned build; bucket-cardinality growth remains a separate concern.
- if let Some(aggs_def) = &request.aggs {
- if fast_aggs::agg_tree_mentions_meta(aggs_def) {
- if let Some(g) = crate::governor::global() {
- if g.query_memory_enabled() {
- // Full source clone + `_id`/`_index`/`_seq_no` injection
- // per buffered doc; a conservative per-doc figure (the
- // window guard above bills 1 KiB/hit, an owned agg clones
- // the whole source).
- const EST_BYTES_PER_OWNED_DOC: u64 = 2048;
- let est = (self.memtable.doc_count() as u64)
- .saturating_mul(EST_BYTES_PER_OWNED_DOC);
- g.check_query_alloc(est, "aggregation owned-source materialisation")
- .map_err(EngineError::Common)?;
- }
- }
- }
- }
-
// Determine the timeout: use the request-level timeout if set, otherwise
// fall back to the default of 30 seconds.
let timeout_ms = request.timeout_ms.unwrap_or(30_000);
@@ -18145,12 +18118,13 @@ impl Index {
// flushed the memtable to segments.
let need_full_corpus = precomputed_aggs.is_none() && request.aggs.is_some();
// #464: the brute agg corpus below deep-clones EVERY memtable AND
- // segment source into owned `Value`s. It is reached by ANY agg that
- // bailed the fast path — percentiles / date_histogram / cardinality have
- // no columnar path at all — so the earlier `agg_tree_mentions_meta` guard
- // (which only sees the fast-path `MemDocs::Owned` clone) missed the whole
- // family. Bound the full-corpus materialisation here too, before the
- // clone, counting memtable AND flushed-segment docs.
+ // segment source into owned `Value`s — the allocation that escaped
+ // `max_query_memory_mb` (the window guard bills `from + size` hits, which
+ // is 0 for `size:0 + aggs`). This path is reached by ANY agg that bails
+ // the columnar fast path: a meta-observing `top_hits`, and equally the
+ // no-meta families with no columnar path at all (percentiles /
+ // date_histogram / cardinality). Bound the full-corpus materialisation
+ // here, before the clone, counting memtable AND flushed-segment docs.
if need_full_corpus {
if let Some(g) = crate::governor::global() {
if g.query_memory_enabled() {
diff --git a/engine/crates/xerj-engine/tests/agg_memory_budget.rs b/engine/crates/xerj-engine/tests/agg_memory_budget.rs
index 8f9579226..3eca5ae8b 100644
--- a/engine/crates/xerj-engine/tests/agg_memory_budget.rs
+++ b/engine/crates/xerj-engine/tests/agg_memory_budget.rs
@@ -2,18 +2,23 @@
//!
//! The per-query memory guard (`governor::check_query_alloc`) is called only for
//! the search result window (`from + size` hits). An aggregation request runs
-//! with `size: 0`, so that estimate is 0 and the guard passes — yet a
-//! meta-observing agg (`top_hits`, or one targeting `_id`/`_index`/`_seq_no`)
-//! deep-clones EVERY buffered memtable document to build its `MemDocs::Owned`
-//! view (`fast_aggs.rs`). Under a bulk writer the memtable holds 10^4-10^5 docs,
-//! so a single agg allocated 5.2x the configured limit with the guard none the
-//! wiser.
+//! with `size: 0`, so that estimate is 0 and the guard passes — yet the brute
+//! full-corpus agg path deep-clones EVERY memtable (and segment) source into an
+//! owned `Value` to feed `run_aggs_with_all`. Under a bulk writer the memtable
+//! holds 10^4-10^5 docs, so a single agg allocated multiples of the configured
+//! limit with the guard none the wiser.
+//!
+//! This path is reached by ANY agg that bails the columnar fast path: a
+//! meta-observing `top_hits` (which needs `_id`/source per bucket), and equally
+//! the no-meta families with no columnar path at all — `percentiles`,
+//! `date_histogram`, `cardinality`. The fix estimates the corpus materialisation
+//! (memtable + flushed-segment docs) up front and rejects before the clone.
//!
//! This test installs a low `max_query_memory_mb`, fills the memtable with more
-//! documents than that budget can materialise, and asserts a `size: 0`
-//! `top_hits` agg is REJECTED (the guard now estimates the owned-source
-//! materialisation and trips before the allocation). Fail-before: without the
-//! agg-path guard the same request returns 200 — the whole point of #464.
+//! documents than that budget can materialise, and asserts BOTH a `top_hits`
+//! agg (meta-observing) and a `percentiles` agg (no meta fields) are REJECTED —
+//! the single corpus guard must bind the whole family, not just meta aggs.
+//! Fail-before: without the guard either request returns 200 — the point of #464.
//!
//! Own test binary on purpose: `Engine::new` installs `max_query_memory_mb` in
//! the process-wide governor static, so no other suite may observe this budget.
@@ -25,11 +30,13 @@ use xerj_common::types::Schema;
use xerj_engine::Engine;
use xerj_query::parse_request;
-/// 1 MiB budget. The owned-source estimate is `memtable_docs * per-doc bytes`;
-/// well under a thousand buffered docs already exceeds this.
+/// 1 MiB budget. The corpus estimate is `corpus_docs * per-doc bytes`; well
+/// under a thousand buffered docs already exceeds this.
const BUDGET_MB: u64 = 1;
-/// Enough buffered docs that the owned-source materialisation estimate clears
-/// the 1 MiB budget, but few enough to stay in the memtable (no flush).
+/// Enough buffered docs that the corpus materialisation estimate clears the
+/// 1 MiB budget, but few enough to stay in the memtable (no flush) and below
+/// `FAST_AGG_MIN_DOCS` — so every agg here takes the brute full-corpus path the
+/// guard protects.
const DOCS: usize = 900;
fn low_budget_engine(dir: &TempDir) -> Engine {
@@ -40,7 +47,7 @@ fn low_budget_engine(dir: &TempDir) -> Engine {
}
#[tokio::test]
-async fn top_hits_aggregation_is_bound_by_max_query_memory_mb() {
+async fn aggregation_corpus_is_bound_by_max_query_memory_mb() {
let dir = TempDir::new().unwrap();
let engine = low_budget_engine(&dir);
engine.create_index("aggmem", Schema::empty()).unwrap();
@@ -67,9 +74,10 @@ async fn top_hits_aggregation_is_bound_by_max_query_memory_mb() {
"a plain size:0 search must not be blocked by the query-memory budget (#464)"
);
- // A `size:0` `top_hits` agg forces `needs_owned_mem` → the memtable
- // deep-clone of all DOCS docs, whose estimate exceeds the 1 MiB budget.
- let agg = parse_request(&json!({
+ // A `size:0` `top_hits` agg (meta-observing) bails the fast path to the
+ // brute full-corpus assembly, which deep-clones all DOCS memtable sources —
+ // an estimate that exceeds the 1 MiB budget.
+ let top_hits = parse_request(&json!({
"query": { "match_all": {} },
"size": 0,
"aggs": {
@@ -80,19 +88,16 @@ async fn top_hits_aggregation_is_bound_by_max_query_memory_mb() {
}
}))
.unwrap();
-
- let result = idx.search(&agg).await;
assert!(
- result.is_err(),
+ idx.search(&top_hits).await.is_err(),
"a top_hits agg over {DOCS} buffered docs must be rejected by \
- max_query_memory_mb={BUDGET_MB}MB — the owned-source deep-clone allocates \
+ max_query_memory_mb={BUDGET_MB}MB — the full-corpus deep-clone allocates \
far beyond the budget (#464). Got Ok, so the agg path bypassed the guard."
);
- // A `percentiles` agg has NO columnar fast path — it bails to the brute
- // full-corpus assembly, which deep-clones every memtable + segment source
- // into owned `Value`s. It does NOT mention meta fields, so the meta-keyed
- // guard alone missed this whole family; the corpus guard must bind it too.
+ // A `percentiles` agg has NO columnar fast path and mentions NO meta field —
+ // it bails to the SAME brute full-corpus assembly. It proves the guard binds
+ // the whole agg family, not just meta-observing aggs (#464).
let percentiles = parse_request(&json!({
"query": { "match_all": {} },
"size": 0,