Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): reject avg/sum/min/max/stats on a non-numeric field like ES (#2913)

Amartya Vikram Singh426336bc0ff7
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 09fe4c615..287c772f0 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -6017,6 +6017,248 @@ fn collect_named_query_scores(
}
}
+/// The mapped `type` of `field` in one index's mapping, following object
+/// `properties` and multi-field `fields` (`name.raw`). `None` when the field
+/// is not mapped there.
+fn mapped_field_type(mapping: &Value, field: &str) -> Option<String> {
+ fn walk(props: &serde_json::Map<String, Value>, segs: &[&str]) -> Option<String> {
+ // A literal dotted key (`"a.b": {...}`) wins, as it does in ES.
+ for split in (1..=segs.len()).rev() {
+ let key = segs[..split].join(".");
+ let Some(node) = props.get(&key) else {
+ continue;
+ };
+ let rest = &segs[split..];
+ if rest.is_empty() {
+ return Some(
+ node.get("type")
+ .and_then(Value::as_str)
+ .unwrap_or("object")
+ .to_string(),
+ );
+ }
+ for child in ["properties", "fields"] {
+ if let Some(found) = node
+ .get(child)
+ .and_then(Value::as_object)
+ .and_then(|p| walk(p, rest))
+ {
+ return Some(found);
+ }
+ }
+ }
+ None
+ }
+ let props = mapping
+ .pointer("/mappings/properties")
+ .or_else(|| mapping.pointer("/properties"))
+ .and_then(Value::as_object)?;
+ let segs: Vec<&str> = field.split('.').collect();
+ walk(props, &segs)
+}
+
+/// A numeric metric aggregation (`avg`, `sum`, `min`, `max`, `stats`,
+/// `extended_stats`) on a field whose mapped type has no numeric values
+/// source. ES rejects these on every shard with `Field [f] of type [t] is
+/// not supported for aggregation [avg]` (`ValuesSourceRegistry#getAggregator`)
+/// and, when every shard fails, answers 400. XERJ answered 200 with a null
+/// value, so a typo'd field looked like an empty index.
+///
+/// Only types ES never accepts for these aggregations are listed; a field that
+/// is unmapped, numeric, `date` or `boolean` in at least one target index is
+/// left alone, since ES would answer 200 (or a partial-shard 200) there.
+fn find_unsupported_metric_agg_field(
+ aggs: Option<&Value>,
+ state: &AppState,
+ indices: &[String],
+) -> Option<String> {
+ const NUMERIC_METRICS: &[&str] = &["avg", "sum", "min", "max", "stats", "extended_stats"];
+ const NON_NUMERIC_TYPES: &[&str] = &[
+ "keyword",
+ "constant_keyword",
+ "wildcard",
+ "ip",
+ "geo_point",
+ "version",
+ ];
+ fn walk(aggs: &serde_json::Map<String, Value>, mappings: &[Value]) -> Option<String> {
+ for def in aggs.values() {
+ let Some(obj) = def.as_object() else {
+ continue;
+ };
+ for (kind, params) in obj {
+ let field = params.get("field").and_then(Value::as_str);
+ if let Some(field) = field.filter(|_| NUMERIC_METRICS.contains(&kind.as_str())) {
+ let types: Vec<Option<String>> = mappings
+ .iter()
+ .map(|m| mapped_field_type(m, field))
+ .collect();
+ let rejected = |t: &Option<String>| {
+ t.as_deref().is_some_and(|t| NON_NUMERIC_TYPES.contains(&t))
+ };
+ if !types.is_empty() && types.iter().all(rejected) {
+ let ty = types[0].as_deref().unwrap_or_default();
+ return Some(format!(
+ "Field [{field}] of type [{ty}] is not supported for \
+ aggregation [{kind}]"
+ ));
+ }
+ }
+ }
+ if let Some(found) = obj
+ .get("aggs")
+ .or_else(|| obj.get("aggregations"))
+ .and_then(Value::as_object)
+ .and_then(|sub| walk(sub, mappings))
+ {
+ return Some(found);
+ }
+ }
+ None
+ }
+ let aggs = aggs?.as_object()?;
+ let mappings: Vec<Value> = indices
+ .iter()
+ .map(|ix| {
+ state
+ .engine
+ .index_mappings
+ .get(ix.as_str())
+ .map(|m| m.clone())
+ })
+ .collect::<Option<_>>()?;
+ walk(aggs, &mappings)
+}
+
+#[cfg(test)]
+mod metric_agg_field_type_tests {
+ use super::*;
+ use axum::{body::Body, http::Request};
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ async fn send(app: &axum::Router, method: &str, uri: &str, body: Value) -> (StatusCode, Value) {
+ let resp = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(uri)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = resp.status();
+ let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (status, serde_json::from_slice(&bytes).unwrap())
+ }
+
+ async fn app_with_indices() -> axum::Router {
+ let app = crate::router::build_es_compat_router(test_state());
+ let mappings = json!({"mappings": {"properties": {
+ "n": {"type": "integer"},
+ "s": {"type": "keyword"},
+ "ip": {"type": "ip"},
+ "name": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
+ "s_alias": {"type": "alias", "path": "s"},
+ }}});
+ let (status, body) = send(&app, "PUT", "/t", mappings).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = send(
+ &app,
+ "PUT",
+ "/u",
+ json!({"mappings": {"properties": {"s": {"type": "long"}}}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let doc = json!({"n": 1, "s": "foo", "ip": "1.2.3.4", "name": "x"});
+ let (status, body) = send(&app, "PUT", "/t/_doc/1?refresh=true", doc).await;
+ assert!(status.is_success(), "{body}");
+ app
+ }
+
+ /// FAIL-BEFORE: each of these answered 200 with `"value": null` (and
+ /// `sum` with `-0.0`), so a metric on the wrong field looked like an
+ /// empty index. ES 8.13 fails every shard and answers 400.
+ #[tokio::test]
+ async fn numeric_metric_on_a_non_numeric_field_is_a_400() {
+ let app = app_with_indices().await;
+ for (aggs, reason) in [
+ (
+ json!({"a": {"avg": {"field": "s"}}}),
+ "Field [s] of type [keyword] is not supported for aggregation [avg]",
+ ),
+ (
+ json!({"a": {"max": {"field": "ip"}}}),
+ "Field [ip] of type [ip] is not supported for aggregation [max]",
+ ),
+ (
+ json!({"a": {"sum": {"field": "name.raw"}}}),
+ "Field [name.raw] of type [keyword] is not supported for aggregation [sum]",
+ ),
+ (
+ json!({"a": {"stats": {"field": "s_alias"}}}),
+ "Field [s] of type [keyword] is not supported for aggregation [stats]",
+ ),
+ (
+ json!({"a": {"terms": {"field": "s"}, "aggs": {"b": {"min": {"field": "s"}}}}}),
+ "Field [s] of type [keyword] is not supported for aggregation [min]",
+ ),
+ ] {
+ let (status, body) =
+ send(&app, "POST", "/t/_search?size=0", json!({ "aggs": aggs })).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{aggs} -> {body}");
+ assert_eq!(body["error"]["type"], "search_phase_execution_exception");
+ assert_eq!(
+ body["error"]["root_cause"][0]["type"],
+ "illegal_argument_exception"
+ );
+ assert_eq!(body["error"]["root_cause"][0]["reason"], reason, "{aggs}");
+ }
+ }
+
+ /// Requests ES answers 200 must keep answering 200: numeric fields,
+ /// unmapped fields, and a field that is numeric in at least one target.
+ #[tokio::test]
+ async fn numeric_unmapped_and_mixed_targets_still_answer() {
+ let app = app_with_indices().await;
+ for (uri, aggs) in [
+ ("/t/_search?size=0", json!({"a": {"avg": {"field": "n"}}})),
+ (
+ "/t/_search?size=0",
+ json!({"a": {"avg": {"field": "nope"}}}),
+ ),
+ ("/t,u/_search?size=0", json!({"a": {"avg": {"field": "s"}}})),
+ (
+ "/t/_search?size=0",
+ json!({"a": {"value_count": {"field": "s"}}}),
+ ),
+ ] {
+ let (status, body) = send(&app, "POST", uri, json!({ "aggs": aggs })).await;
+ assert_eq!(status, StatusCode::OK, "{uri} {aggs} -> {body}");
+ }
+ }
+}
+
/// For each `{"exists": {"field": X}}` clause, check whether `X` is queryable
/// in *any* of the participating indices. A field is unqueryable if the
/// mapping declares `index: false` AND `doc_values: false` (or the field
@@ -15647,6 +15889,15 @@ async fn search_impl(
// below — see `per_index_collapse_rewrite`).
}
+ // A numeric metric on a keyword/ip/... field. Checked after the alias
+ // rewrite above so an alias is judged by the type of the field it names.
+ let metric_aggs = body.aggs.as_ref().or(body.aggregations.as_ref());
+ if let Some(reason) =
+ find_unsupported_metric_agg_field(metric_aggs, &state, &participating_indices)
+ {
+ return search_shard_validation_error(&reason);
+ }
+
// Enforce per-index `index.disable_sequence_numbers` setting:
// reject queries/sorts that touch `_seq_no` with ES's specific
// error messages.
diff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txt
index 42f53a275..135188f1c 100644
--- a/engine/tests/es-compat-yaml/catch-lenient.txt
+++ b/engine/tests/es-compat-yaml/catch-lenient.txt
@@ -8,7 +8,6 @@
# make a new failure go away; a new failure is a regression.
#
# Format: <path under yaml/> · <test name>
-aggregations/avg_metric.yml · Aggregating wrong datatype test
aggregations/bucket_script.yml · bad script
aggregations/bucket_selector.yml · bad script
aggregations/composite.yml · Composite aggregation with invalid format
@@ -19,8 +18,6 @@ aggregations/matrix_stats_multi_value_field.yml · With script params
aggregations/matrix_stats_single_value_field.yml · With script
aggregations/matrix_stats_single_value_field.yml · With script params
aggregations/max_buckets.yml · Max bucket
-aggregations/max_metric.yml · Aggregating wrong datatype test
-aggregations/min_metric.yml · Aggregating wrong datatype test
aggregations/nested.yml · Illegal reverse nested aggregation to a child nested object
aggregations/percentiles_hdr_metric.yml · Invalid params test
aggregations/percentiles_tdigest_metric.yml · Invalid params test
@@ -44,7 +41,6 @@ aggregations/scripted_metric_allow_list.yml · stored reduce_script must be allo
aggregations/sig_terms.yml · IP test
aggregations/sig_terms.yml · Misspelled fields get "did you mean"
aggregations/stats_metric_fail_formatting.yml · fail formatting
-aggregations/sum_metric.yml · Aggregating wrong datatype test
aggregations/terms.yml · IP test
aggregations/top_hits.yml · having
aggregations/weighted_avg.yml · Multi weight field not allowed