Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(aggs): multi_terms raises too_many_buckets past the cap; agg bucket-cap errors are real HTTP 400s (RC4-W2 item 8)

xerj-org573bac506f93
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 5819af4ab..7645dd4d6 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -9654,6 +9654,62 @@ pub async fn search(
}
}
+ // An aggregation runner can embed a hard failure — e.g. too_many_buckets
+ // from histogram / date_histogram / multi_terms — as
+ // `{"error": "<reason>", "__error_status__": 400}` inside its slot of the
+ // result tree. ES surfaces these as a real HTTP 400
+ // search_phase_execution_exception caused by too_many_buckets_exception
+ // (verified against live ES 8.13.4); previously xerj leaked the raw
+ // marker object inline under HTTP 200.
+ if let Some(aggs) = merged_aggs.as_ref() {
+ fn find_embedded_agg_error(v: &Value) -> Option<String> {
+ match v {
+ Value::Object(obj) => {
+ if let (Some(Value::String(msg)), Some(st)) =
+ (obj.get("error"), obj.get("__error_status__"))
+ {
+ if st.is_number() {
+ return Some(msg.clone());
+ }
+ }
+ obj.values().find_map(find_embedded_agg_error)
+ }
+ Value::Array(arr) => arr.iter().find_map(find_embedded_agg_error),
+ _ => None,
+ }
+ }
+ if let Some(reason) = find_embedded_agg_error(aggs) {
+ if reason.contains("Trying to create too many buckets") {
+ let max_b: u64 = reason
+ .split('[')
+ .nth(1)
+ .and_then(|s| s.split(']').next())
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(65_536);
+ return (
+ StatusCode::BAD_REQUEST,
+ Json(json!({
+ "error": {
+ "root_cause": [],
+ "type": "search_phase_execution_exception",
+ "reason": "",
+ "phase": "fetch",
+ "grouped": true,
+ "failed_shards": [],
+ "caused_by": {
+ "type": "too_many_buckets_exception",
+ "reason": reason,
+ "max_buckets": max_b
+ }
+ },
+ "status": 400
+ })),
+ )
+ .into_response();
+ }
+ }
+ }
+
// Apply typed_keys: prefix each aggregation name with its type.
if let Some(mut aggs) = merged_aggs {
strip_internal_tracking(&mut aggs);
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 939e10d23..df4adbf55 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -4555,7 +4555,8 @@ fn run_date_histogram(
"error": format!(
"Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
MAX_BUCKETS
- )
+ ),
+ "__error_status__": 400u32,
});
}
probe = if use_calendar {
@@ -8948,6 +8949,19 @@ fn run_multi_terms(
.collect();
if bucket_map.contains_key(&key) || bucket_map.len() < bucket_cap {
bucket_map.entry(key).or_default().push(i);
+ } else {
+ // Materialising another bucket would blow the search.max_buckets
+ // cap. Raise too_many_buckets — exactly like date_histogram /
+ // histogram already do — instead of silently dropping every
+ // bucket past the cap (which corrupted both the top-N selection
+ // and the doc counts).
+ return json!({
+ "error": format!(
+ "Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
+ bucket_cap
+ ),
+ "__error_status__": 400u32,
+ });
}
}
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 70486d982..d1ad14e14 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -4134,7 +4134,8 @@ impl<'a> FastCtx<'a> {
"error": format!(
"Trying to create too many buckets. Must be less than or equal to: [{}] but this number of buckets was exceeded. This limit can be set by changing the [search.max_buckets] cluster level setting.",
MAX_BUCKETS
- )
+ ),
+ "__error_status__": 400u32,
}));
}
probe = next_bucket(probe);