AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): reject avg/sum/min/max/stats on a non-numeric field like ES (#2913)
Amartya Vikram Singh426336bc0ff7
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 09fe4c615..287c772f0 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -6017,6 +6017,248 @@ fn collect_named_query_scores(}}+/// The mapped `type` of `field` in one index's mapping, following object+/// `properties` and multi-field `fields` (`name.raw`). `None` when the field+/// is not mapped there.+fn mapped_field_type(mapping: &Value, field: &str) -> Option<String> {+ fn walk(props: &serde_json::Map<String, Value>, segs: &[&str]) -> Option<String> {+ // A literal dotted key (`"a.b": {...}`) wins, as it does in ES.+ for split in (1..=segs.len()).rev() {+ let key = segs[..split].join(".");+ let Some(node) = props.get(&key) else {+ continue;+ };+ let rest = &segs[split..];+ if rest.is_empty() {+ return Some(+ node.get("type")+ .and_then(Value::as_str)+ .unwrap_or("object")+ .to_string(),+ );+ }+ for child in ["properties", "fields"] {+ if let Some(found) = node+ .get(child)+ .and_then(Value::as_object)+ .and_then(|p| walk(p, rest))+ {+ return Some(found);+ }+ }+ }+ None+ }+ let props = mapping+ .pointer("/mappings/properties")+ .or_else(|| mapping.pointer("/properties"))+ .and_then(Value::as_object)?;+ let segs: Vec<&str> = field.split('.').collect();+ walk(props, &segs)+}++/// A numeric metric aggregation (`avg`, `sum`, `min`, `max`, `stats`,+/// `extended_stats`) on a field whose mapped type has no numeric values+/// source. ES rejects these on every shard with `Field [f] of type [t] is+/// not supported for aggregation [avg]` (`ValuesSourceRegistry#getAggregator`)+/// and, when every shard fails, answers 400. XERJ answered 200 with a null+/// value, so a typo'd field looked like an empty index.+///+/// Only types ES never accepts for these aggregations are listed; a field that+/// is unmapped, numeric, `date` or `boolean` in at least one target index is+/// left alone, since ES would answer 200 (or a partial-shard 200) there.+fn find_unsupported_metric_agg_field(+ aggs: Option<&Value>,+ state: &AppState,+ indices: &[String],+) -> Option<String> {+ const NUMERIC_METRICS: &[&str] = &["avg", "sum", "min", "max", "stats", "extended_stats"];+ const NON_NUMERIC_TYPES: &[&str] = &[+ "keyword",+ "constant_keyword",+ "wildcard",+ "ip",+ "geo_point",+ "version",+ ];+ fn walk(aggs: &serde_json::Map<String, Value>, mappings: &[Value]) -> Option<String> {+ for def in aggs.values() {+ let Some(obj) = def.as_object() else {+ continue;+ };+ for (kind, params) in obj {+ let field = params.get("field").and_then(Value::as_str);+ if let Some(field) = field.filter(|_| NUMERIC_METRICS.contains(&kind.as_str())) {+ let types: Vec<Option<String>> = mappings+ .iter()+ .map(|m| mapped_field_type(m, field))+ .collect();+ let rejected = |t: &Option<String>| {+ t.as_deref().is_some_and(|t| NON_NUMERIC_TYPES.contains(&t))+ };+ if !types.is_empty() && types.iter().all(rejected) {+ let ty = types[0].as_deref().unwrap_or_default();+ return Some(format!(+ "Field [{field}] of type [{ty}] is not supported for \+ aggregation [{kind}]"+ ));+ }+ }+ }+ if let Some(found) = obj+ .get("aggs")+ .or_else(|| obj.get("aggregations"))+ .and_then(Value::as_object)+ .and_then(|sub| walk(sub, mappings))+ {+ return Some(found);+ }+ }+ None+ }+ let aggs = aggs?.as_object()?;+ let mappings: Vec<Value> = indices+ .iter()+ .map(|ix| {+ state+ .engine+ .index_mappings+ .get(ix.as_str())+ .map(|m| m.clone())+ })+ .collect::<Option<_>>()?;+ walk(aggs, &mappings)+}++#[cfg(test)]+mod metric_agg_field_type_tests {+ use super::*;+ use axum::{body::Body, http::Request};+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ async fn send(app: &axum::Router, method: &str, uri: &str, body: Value) -> (StatusCode, Value) {+ let resp = app+ .clone()+ .oneshot(+ Request::builder()+ .method(method)+ .uri(uri)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .unwrap(),+ )+ .await+ .unwrap();+ let status = resp.status();+ let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, serde_json::from_slice(&bytes).unwrap())+ }++ async fn app_with_indices() -> axum::Router {+ let app = crate::router::build_es_compat_router(test_state());+ let mappings = json!({"mappings": {"properties": {+ "n": {"type": "integer"},+ "s": {"type": "keyword"},+ "ip": {"type": "ip"},+ "name": {"type": "text", "fields": {"raw": {"type": "keyword"}}},+ "s_alias": {"type": "alias", "path": "s"},+ }}});+ let (status, body) = send(&app, "PUT", "/t", mappings).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = send(+ &app,+ "PUT",+ "/u",+ json!({"mappings": {"properties": {"s": {"type": "long"}}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let doc = json!({"n": 1, "s": "foo", "ip": "1.2.3.4", "name": "x"});+ let (status, body) = send(&app, "PUT", "/t/_doc/1?refresh=true", doc).await;+ assert!(status.is_success(), "{body}");+ app+ }++ /// FAIL-BEFORE: each of these answered 200 with `"value": null` (and+ /// `sum` with `-0.0`), so a metric on the wrong field looked like an+ /// empty index. ES 8.13 fails every shard and answers 400.+ #[tokio::test]+ async fn numeric_metric_on_a_non_numeric_field_is_a_400() {+ let app = app_with_indices().await;+ for (aggs, reason) in [+ (+ json!({"a": {"avg": {"field": "s"}}}),+ "Field [s] of type [keyword] is not supported for aggregation [avg]",+ ),+ (+ json!({"a": {"max": {"field": "ip"}}}),+ "Field [ip] of type [ip] is not supported for aggregation [max]",+ ),+ (+ json!({"a": {"sum": {"field": "name.raw"}}}),+ "Field [name.raw] of type [keyword] is not supported for aggregation [sum]",+ ),+ (+ json!({"a": {"stats": {"field": "s_alias"}}}),+ "Field [s] of type [keyword] is not supported for aggregation [stats]",+ ),+ (+ json!({"a": {"terms": {"field": "s"}, "aggs": {"b": {"min": {"field": "s"}}}}}),+ "Field [s] of type [keyword] is not supported for aggregation [min]",+ ),+ ] {+ let (status, body) =+ send(&app, "POST", "/t/_search?size=0", json!({ "aggs": aggs })).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{aggs} -> {body}");+ assert_eq!(body["error"]["type"], "search_phase_execution_exception");+ assert_eq!(+ body["error"]["root_cause"][0]["type"],+ "illegal_argument_exception"+ );+ assert_eq!(body["error"]["root_cause"][0]["reason"], reason, "{aggs}");+ }+ }++ /// Requests ES answers 200 must keep answering 200: numeric fields,+ /// unmapped fields, and a field that is numeric in at least one target.+ #[tokio::test]+ async fn numeric_unmapped_and_mixed_targets_still_answer() {+ let app = app_with_indices().await;+ for (uri, aggs) in [+ ("/t/_search?size=0", json!({"a": {"avg": {"field": "n"}}})),+ (+ "/t/_search?size=0",+ json!({"a": {"avg": {"field": "nope"}}}),+ ),+ ("/t,u/_search?size=0", json!({"a": {"avg": {"field": "s"}}})),+ (+ "/t/_search?size=0",+ json!({"a": {"value_count": {"field": "s"}}}),+ ),+ ] {+ let (status, body) = send(&app, "POST", uri, json!({ "aggs": aggs })).await;+ assert_eq!(status, StatusCode::OK, "{uri} {aggs} -> {body}");+ }+ }+}+/// For each `{"exists": {"field": X}}` clause, check whether `X` is queryable/// in *any* of the participating indices. A field is unqueryable if the/// mapping declares `index: false` AND `doc_values: false` (or the field@@ -15647,6 +15889,15 @@ async fn search_impl(// below — see `per_index_collapse_rewrite`).}+ // A numeric metric on a keyword/ip/... field. Checked after the alias+ // rewrite above so an alias is judged by the type of the field it names.+ let metric_aggs = body.aggs.as_ref().or(body.aggregations.as_ref());+ if let Some(reason) =+ find_unsupported_metric_agg_field(metric_aggs, &state, &participating_indices)+ {+ return search_shard_validation_error(&reason);+ }+// Enforce per-index `index.disable_sequence_numbers` setting:// reject queries/sorts that touch `_seq_no` with ES's specific// error messages.diff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txtindex 42f53a275..135188f1c 100644--- a/engine/tests/es-compat-yaml/catch-lenient.txt+++ b/engine/tests/es-compat-yaml/catch-lenient.txt@@ -8,7 +8,6 @@# make a new failure go away; a new failure is a regression.## Format: <path under yaml/> · <test name>-aggregations/avg_metric.yml · Aggregating wrong datatype testaggregations/bucket_script.yml · bad scriptaggregations/bucket_selector.yml · bad scriptaggregations/composite.yml · Composite aggregation with invalid format@@ -19,8 +18,6 @@ aggregations/matrix_stats_multi_value_field.yml · With script paramsaggregations/matrix_stats_single_value_field.yml · With scriptaggregations/matrix_stats_single_value_field.yml · With script paramsaggregations/max_buckets.yml · Max bucket-aggregations/max_metric.yml · Aggregating wrong datatype test-aggregations/min_metric.yml · Aggregating wrong datatype testaggregations/nested.yml · Illegal reverse nested aggregation to a child nested objectaggregations/percentiles_hdr_metric.yml · Invalid params testaggregations/percentiles_tdigest_metric.yml · Invalid params test@@ -44,7 +41,6 @@ aggregations/scripted_metric_allow_list.yml · stored reduce_script must be alloaggregations/sig_terms.yml · IP testaggregations/sig_terms.yml · Misspelled fields get "did you mean"aggregations/stats_metric_fail_formatting.yml · fail formatting-aggregations/sum_metric.yml · Aggregating wrong datatype testaggregations/terms.yml · IP testaggregations/top_hits.yml · havingaggregations/weighted_avg.yml · Multi weight field not allowed