Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): bind aggregation source materialisation to max_query_memory_mb, both agg paths (#464)

xerj-org6d236df7a760
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index df4308674..66a6a9fa7 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -5132,7 +5132,7 @@ fn compile_top_pred(filter: &Value) -> Option<Pred> {
/// costs an extra clone, never correctness — so this deliberately mirrors
/// the substring semantics (`top_hits` anywhere; the meta fields as exact
/// key/string tokens, i.e. the serialized quoted form).
-fn agg_tree_mentions_meta(v: &Value) -> bool {
+pub(super) fn agg_tree_mentions_meta(v: &Value) -> bool {
fn is_meta(s: &str) -> bool {
s.contains("top_hits") || s == "_id" || s == "_index" || s == "_seq_no"
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index fc6d39858..563359e5c 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -13863,6 +13863,33 @@ impl Index {
}
}
+ // #464: the guard above only covers the search RESULT WINDOW. An
+ // aggregation runs with `size: 0`, so that estimate is 0 — yet a
+ // meta-observing agg (`top_hits`, or one targeting `_id`/`_index`/
+ // `_seq_no`) deep-clones EVERY buffered memtable document to build its
+ // owned-source view (`fast_aggs::MemDocs::Owned`), which under a bulk
+ // writer is 10^4-10^5 full-source clones. That allocation escaped
+ // `max_query_memory_mb` entirely. Estimate it here — before either the
+ // fast or the brute agg path runs — and reject up-front. Bounds the
+ // owned build; bucket-cardinality growth remains a separate concern.
+ if let Some(aggs_def) = &request.aggs {
+ if fast_aggs::agg_tree_mentions_meta(aggs_def) {
+ if let Some(g) = crate::governor::global() {
+ if g.query_memory_enabled() {
+ // Full source clone + `_id`/`_index`/`_seq_no` injection
+ // per buffered doc; a conservative per-doc figure (the
+ // window guard above bills 1 KiB/hit, an owned agg clones
+ // the whole source).
+ const EST_BYTES_PER_OWNED_DOC: u64 = 2048;
+ let est = (self.memtable.doc_count() as u64)
+ .saturating_mul(EST_BYTES_PER_OWNED_DOC);
+ g.check_query_alloc(est, "aggregation owned-source materialisation")
+ .map_err(EngineError::Common)?;
+ }
+ }
+ }
+ }
+
// Determine the timeout: use the request-level timeout if set, otherwise
// fall back to the default of 30 seconds.
let timeout_ms = request.timeout_ms.unwrap_or(30_000);
@@ -18117,6 +18144,32 @@ impl Index {
// `min_doc_count:0` background, so it must exist after `_refresh` has
// flushed the memtable to segments.
let need_full_corpus = precomputed_aggs.is_none() && request.aggs.is_some();
+ // #464: the brute agg corpus below deep-clones EVERY memtable AND
+ // segment source into owned `Value`s. It is reached by ANY agg that
+ // bailed the fast path — percentiles / date_histogram / cardinality have
+ // no columnar path at all — so the earlier `agg_tree_mentions_meta` guard
+ // (which only sees the fast-path `MemDocs::Owned` clone) missed the whole
+ // family. Bound the full-corpus materialisation here too, before the
+ // clone, counting memtable AND flushed-segment docs.
+ if need_full_corpus {
+ if let Some(g) = crate::governor::global() {
+ if g.query_memory_enabled() {
+ const EST_BYTES_PER_OWNED_DOC: u64 = 2048;
+ let seg_docs: u64 = self
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|m| m.doc_count)
+ .sum();
+ let est = (self.memtable.doc_count() as u64)
+ .saturating_add(seg_docs)
+ .saturating_mul(EST_BYTES_PER_OWNED_DOC);
+ g.check_query_alloc(est, "aggregation corpus materialisation")
+ .map_err(EngineError::Common)?;
+ }
+ }
+ }
let all_docs: Vec<Value> = if need_full_corpus {
// `_id` is injected onto each source so `top_hits` / `_id`-keyed
// aggs over the corpus still work (the fast path never provided it).
diff --git a/engine/crates/xerj-engine/tests/agg_memory_budget.rs b/engine/crates/xerj-engine/tests/agg_memory_budget.rs
new file mode 100644
index 000000000..8f9579226
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/agg_memory_budget.rs
@@ -0,0 +1,107 @@
+//! Issue #464: `max_query_memory_mb` does not cover aggregation memory.
+//!
+//! The per-query memory guard (`governor::check_query_alloc`) is called only for
+//! the search result window (`from + size` hits). An aggregation request runs
+//! with `size: 0`, so that estimate is 0 and the guard passes — yet a
+//! meta-observing agg (`top_hits`, or one targeting `_id`/`_index`/`_seq_no`)
+//! deep-clones EVERY buffered memtable document to build its `MemDocs::Owned`
+//! view (`fast_aggs.rs`). Under a bulk writer the memtable holds 10^4-10^5 docs,
+//! so a single agg allocated 5.2x the configured limit with the guard none the
+//! wiser.
+//!
+//! This test installs a low `max_query_memory_mb`, fills the memtable with more
+//! documents than that budget can materialise, and asserts a `size: 0`
+//! `top_hits` agg is REJECTED (the guard now estimates the owned-source
+//! materialisation and trips before the allocation). Fail-before: without the
+//! agg-path guard the same request returns 200 — the whole point of #464.
+//!
+//! Own test binary on purpose: `Engine::new` installs `max_query_memory_mb` in
+//! the process-wide governor static, so no other suite may observe this budget.
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+/// 1 MiB budget. The owned-source estimate is `memtable_docs * per-doc bytes`;
+/// well under a thousand buffered docs already exceeds this.
+const BUDGET_MB: u64 = 1;
+/// Enough buffered docs that the owned-source materialisation estimate clears
+/// the 1 MiB budget, but few enough to stay in the memtable (no flush).
+const DOCS: usize = 900;
+
+fn low_budget_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ config.limits.max_query_memory_mb = BUDGET_MB;
+ Engine::new(config).expect("engine::new installs the governor from config")
+}
+
+#[tokio::test]
+async fn top_hits_aggregation_is_bound_by_max_query_memory_mb() {
+ let dir = TempDir::new().unwrap();
+ let engine = low_budget_engine(&dir);
+ engine.create_index("aggmem", Schema::empty()).unwrap();
+ let idx = engine.get_index("aggmem").unwrap();
+
+ for i in 0..DOCS {
+ idx.index_document(
+ Some(format!("d{i:05}")),
+ json!({
+ "cat": if i % 2 == 0 { "A" } else { "B" },
+ "title": format!("doc number {i}"),
+ "n": i as i64,
+ }),
+ )
+ .await
+ .unwrap();
+ }
+
+ // A control search WITHOUT aggs (size:0) must still pass — its window
+ // estimate is 0, so this isolates the failure to the aggregation path.
+ let control = parse_request(&json!({ "query": { "match_all": {} }, "size": 0 })).unwrap();
+ assert!(
+ idx.search(&control).await.is_ok(),
+ "a plain size:0 search must not be blocked by the query-memory budget (#464)"
+ );
+
+ // A `size:0` `top_hits` agg forces `needs_owned_mem` → the memtable
+ // deep-clone of all DOCS docs, whose estimate exceeds the 1 MiB budget.
+ let agg = parse_request(&json!({
+ "query": { "match_all": {} },
+ "size": 0,
+ "aggs": {
+ "by_cat": {
+ "terms": { "field": "cat", "size": 10 },
+ "aggs": { "top": { "top_hits": { "size": 2, "_source": ["title"] } } }
+ }
+ }
+ }))
+ .unwrap();
+
+ let result = idx.search(&agg).await;
+ assert!(
+ result.is_err(),
+ "a top_hits agg over {DOCS} buffered docs must be rejected by \
+ max_query_memory_mb={BUDGET_MB}MB — the owned-source deep-clone allocates \
+ far beyond the budget (#464). Got Ok, so the agg path bypassed the guard."
+ );
+
+ // A `percentiles` agg has NO columnar fast path — it bails to the brute
+ // full-corpus assembly, which deep-clones every memtable + segment source
+ // into owned `Value`s. It does NOT mention meta fields, so the meta-keyed
+ // guard alone missed this whole family; the corpus guard must bind it too.
+ let percentiles = parse_request(&json!({
+ "query": { "match_all": {} },
+ "size": 0,
+ "aggs": { "p": { "percentiles": { "field": "n" } } }
+ }))
+ .unwrap();
+ assert!(
+ idx.search(&percentiles).await.is_err(),
+ "a percentiles agg (brute full-corpus deep-clone, no meta fields) must \
+ ALSO be bound by max_query_memory_mb — not just meta-observing aggs (#464)"
+ );
+}