AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): bind aggregation source materialisation to max_query_memory_mb, both agg paths (#464)
xerj-org6d236df7a760
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex df4308674..66a6a9fa7 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -5132,7 +5132,7 @@ fn compile_top_pred(filter: &Value) -> Option<Pred> {/// costs an extra clone, never correctness — so this deliberately mirrors/// the substring semantics (`top_hits` anywhere; the meta fields as exact/// key/string tokens, i.e. the serialized quoted form).-fn agg_tree_mentions_meta(v: &Value) -> bool {+pub(super) fn agg_tree_mentions_meta(v: &Value) -> bool {fn is_meta(s: &str) -> bool {s.contains("top_hits") || s == "_id" || s == "_index" || s == "_seq_no"}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex fc6d39858..563359e5c 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -13863,6 +13863,33 @@ impl Index {}}+ // #464: the guard above only covers the search RESULT WINDOW. An+ // aggregation runs with `size: 0`, so that estimate is 0 — yet a+ // meta-observing agg (`top_hits`, or one targeting `_id`/`_index`/+ // `_seq_no`) deep-clones EVERY buffered memtable document to build its+ // owned-source view (`fast_aggs::MemDocs::Owned`), which under a bulk+ // writer is 10^4-10^5 full-source clones. That allocation escaped+ // `max_query_memory_mb` entirely. Estimate it here — before either the+ // fast or the brute agg path runs — and reject up-front. Bounds the+ // owned build; bucket-cardinality growth remains a separate concern.+ if let Some(aggs_def) = &request.aggs {+ if fast_aggs::agg_tree_mentions_meta(aggs_def) {+ if let Some(g) = crate::governor::global() {+ if g.query_memory_enabled() {+ // Full source clone + `_id`/`_index`/`_seq_no` injection+ // per buffered doc; a conservative per-doc figure (the+ // window guard above bills 1 KiB/hit, an owned agg clones+ // the whole source).+ const EST_BYTES_PER_OWNED_DOC: u64 = 2048;+ let est = (self.memtable.doc_count() as u64)+ .saturating_mul(EST_BYTES_PER_OWNED_DOC);+ g.check_query_alloc(est, "aggregation owned-source materialisation")+ .map_err(EngineError::Common)?;+ }+ }+ }+ }+// Determine the timeout: use the request-level timeout if set, otherwise// fall back to the default of 30 seconds.let timeout_ms = request.timeout_ms.unwrap_or(30_000);@@ -18117,6 +18144,32 @@ impl Index {// `min_doc_count:0` background, so it must exist after `_refresh` has// flushed the memtable to segments.let need_full_corpus = precomputed_aggs.is_none() && request.aggs.is_some();+ // #464: the brute agg corpus below deep-clones EVERY memtable AND+ // segment source into owned `Value`s. It is reached by ANY agg that+ // bailed the fast path — percentiles / date_histogram / cardinality have+ // no columnar path at all — so the earlier `agg_tree_mentions_meta` guard+ // (which only sees the fast-path `MemDocs::Owned` clone) missed the whole+ // family. Bound the full-corpus materialisation here too, before the+ // clone, counting memtable AND flushed-segment docs.+ if need_full_corpus {+ if let Some(g) = crate::governor::global() {+ if g.query_memory_enabled() {+ const EST_BYTES_PER_OWNED_DOC: u64 = 2048;+ let seg_docs: u64 = self+ .store+ .snapshot()+ .segments+ .iter()+ .map(|m| m.doc_count)+ .sum();+ let est = (self.memtable.doc_count() as u64)+ .saturating_add(seg_docs)+ .saturating_mul(EST_BYTES_PER_OWNED_DOC);+ g.check_query_alloc(est, "aggregation corpus materialisation")+ .map_err(EngineError::Common)?;+ }+ }+ }let all_docs: Vec<Value> = if need_full_corpus {// `_id` is injected onto each source so `top_hits` / `_id`-keyed// aggs over the corpus still work (the fast path never provided it).diff --git a/engine/crates/xerj-engine/tests/agg_memory_budget.rs b/engine/crates/xerj-engine/tests/agg_memory_budget.rsnew file mode 100644index 000000000..8f9579226--- /dev/null+++ b/engine/crates/xerj-engine/tests/agg_memory_budget.rs@@ -0,0 +1,107 @@+//! Issue #464: `max_query_memory_mb` does not cover aggregation memory.+//!+//! The per-query memory guard (`governor::check_query_alloc`) is called only for+//! the search result window (`from + size` hits). An aggregation request runs+//! with `size: 0`, so that estimate is 0 and the guard passes — yet a+//! meta-observing agg (`top_hits`, or one targeting `_id`/`_index`/`_seq_no`)+//! deep-clones EVERY buffered memtable document to build its `MemDocs::Owned`+//! view (`fast_aggs.rs`). Under a bulk writer the memtable holds 10^4-10^5 docs,+//! so a single agg allocated 5.2x the configured limit with the guard none the+//! wiser.+//!+//! This test installs a low `max_query_memory_mb`, fills the memtable with more+//! documents than that budget can materialise, and asserts a `size: 0`+//! `top_hits` agg is REJECTED (the guard now estimates the owned-source+//! materialisation and trips before the allocation). Fail-before: without the+//! agg-path guard the same request returns 200 — the whole point of #464.+//!+//! Own test binary on purpose: `Engine::new` installs `max_query_memory_mb` in+//! the process-wide governor static, so no other suite may observe this budget.++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;+use xerj_engine::Engine;+use xerj_query::parse_request;++/// 1 MiB budget. The owned-source estimate is `memtable_docs * per-doc bytes`;+/// well under a thousand buffered docs already exceeds this.+const BUDGET_MB: u64 = 1;+/// Enough buffered docs that the owned-source materialisation estimate clears+/// the 1 MiB budget, but few enough to stay in the memtable (no flush).+const DOCS: usize = 900;++fn low_budget_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ config.limits.max_query_memory_mb = BUDGET_MB;+ Engine::new(config).expect("engine::new installs the governor from config")+}++#[tokio::test]+async fn top_hits_aggregation_is_bound_by_max_query_memory_mb() {+ let dir = TempDir::new().unwrap();+ let engine = low_budget_engine(&dir);+ engine.create_index("aggmem", Schema::empty()).unwrap();+ let idx = engine.get_index("aggmem").unwrap();++ for i in 0..DOCS {+ idx.index_document(+ Some(format!("d{i:05}")),+ json!({+ "cat": if i % 2 == 0 { "A" } else { "B" },+ "title": format!("doc number {i}"),+ "n": i as i64,+ }),+ )+ .await+ .unwrap();+ }++ // A control search WITHOUT aggs (size:0) must still pass — its window+ // estimate is 0, so this isolates the failure to the aggregation path.+ let control = parse_request(&json!({ "query": { "match_all": {} }, "size": 0 })).unwrap();+ assert!(+ idx.search(&control).await.is_ok(),+ "a plain size:0 search must not be blocked by the query-memory budget (#464)"+ );++ // A `size:0` `top_hits` agg forces `needs_owned_mem` → the memtable+ // deep-clone of all DOCS docs, whose estimate exceeds the 1 MiB budget.+ let agg = parse_request(&json!({+ "query": { "match_all": {} },+ "size": 0,+ "aggs": {+ "by_cat": {+ "terms": { "field": "cat", "size": 10 },+ "aggs": { "top": { "top_hits": { "size": 2, "_source": ["title"] } } }+ }+ }+ }))+ .unwrap();++ let result = idx.search(&agg).await;+ assert!(+ result.is_err(),+ "a top_hits agg over {DOCS} buffered docs must be rejected by \+ max_query_memory_mb={BUDGET_MB}MB — the owned-source deep-clone allocates \+ far beyond the budget (#464). Got Ok, so the agg path bypassed the guard."+ );++ // A `percentiles` agg has NO columnar fast path — it bails to the brute+ // full-corpus assembly, which deep-clones every memtable + segment source+ // into owned `Value`s. It does NOT mention meta fields, so the meta-keyed+ // guard alone missed this whole family; the corpus guard must bind it too.+ let percentiles = parse_request(&json!({+ "query": { "match_all": {} },+ "size": 0,+ "aggs": { "p": { "percentiles": { "field": "n" } } }+ }))+ .unwrap();+ assert!(+ idx.search(&percentiles).await.is_err(),+ "a percentiles agg (brute full-corpus deep-clone, no meta fields) must \+ ALSO be bound by max_query_memory_mb — not just meta-observing aggs (#464)"+ );+}