Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(aggs): type composite bucket keys from the source field MAPPING (RC4-W2 item 7)

xerj-org22b2bd292fa4
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 7254f6185..5819af4ab 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -5691,6 +5691,150 @@ pub async fn search(
}
}
+ // Annotate every `composite` agg with `__xy_field_types__`: the mapping
+ // type of each of its `terms` sources' fields. The engine types the
+ // composite bucket keys from the SOURCE FIELD MAPPING (keyword → string,
+ // boolean → true/false, numerics → numbers) instead of string-parse
+ // heuristics that corrupted keyword values like "007" into the number 7
+ // and rendered booleans as the strings "true"/"false".
+ {
+ fn gather_composite_terms_fields(v: &Value, out: &mut std::collections::HashSet<String>) {
+ match v {
+ Value::Object(obj) => {
+ if let Some(sources) = obj
+ .get("composite")
+ .and_then(|c| c.get("sources"))
+ .and_then(Value::as_array)
+ {
+ for src in sources {
+ if let Some(src_obj) = src.as_object() {
+ for spec in src_obj.values() {
+ if let Some(f) = spec
+ .get("terms")
+ .and_then(|t| t.get("field"))
+ .and_then(Value::as_str)
+ {
+ out.insert(f.to_string());
+ }
+ }
+ }
+ }
+ }
+ for c in obj.values() {
+ gather_composite_terms_fields(c, out);
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ gather_composite_terms_fields(item, out);
+ }
+ }
+ _ => {}
+ }
+ }
+ let mut wanted: std::collections::HashSet<String> = std::collections::HashSet::new();
+ if let Some(aggs) = body.aggs.as_ref() {
+ gather_composite_terms_fields(aggs, &mut wanted);
+ }
+ if let Some(aggs) = body.aggregations.as_ref() {
+ gather_composite_terms_fields(aggs, &mut wanted);
+ }
+ if !wanted.is_empty() {
+ // Resolve a (possibly dotted / multi-field) path against a
+ // mapping `properties` object: `user.name` walks properties,
+ // `agent.keyword` walks multi-`fields`.
+ fn type_in_props(props: &Value, field: &str) -> Option<String> {
+ let mut cur = props;
+ let mut parts = field.split('.').peekable();
+ while let Some(part) = parts.next() {
+ let spec = cur.get(part)?;
+ if parts.peek().is_none() {
+ return spec.get("type").and_then(Value::as_str).map(str::to_string);
+ }
+ cur = spec.get("properties").or_else(|| spec.get("fields"))?;
+ }
+ None
+ }
+ let mut types: serde_json::Map<String, Value> = serde_json::Map::new();
+ for f in &wanted {
+ for ix in &index_names {
+ if let Some(m) = state.engine.index_mappings.get(*ix) {
+ let mapping = m.clone();
+ let props = mapping
+ .get("mappings")
+ .and_then(|mm| mm.get("properties"))
+ .or_else(|| mapping.get("properties"))
+ .cloned();
+ if let Some(p) = props {
+ if let Some(t) = type_in_props(&p, f) {
+ types.insert(f.clone(), Value::String(t));
+ break;
+ }
+ }
+ }
+ }
+ }
+ if !types.is_empty() {
+ fn annotate(v: &mut Value, types: &serde_json::Map<String, Value>) {
+ match v {
+ Value::Object(obj) => {
+ if let Some(comp) =
+ obj.get_mut("composite").and_then(|c| c.as_object_mut())
+ {
+ if !comp.contains_key("__xy_field_types__") {
+ let mut local = serde_json::Map::new();
+ if let Some(sources) =
+ comp.get("sources").and_then(Value::as_array)
+ {
+ for src in sources {
+ if let Some(src_obj) = src.as_object() {
+ for spec in src_obj.values() {
+ if let Some(f) = spec
+ .get("terms")
+ .and_then(|t| t.get("field"))
+ .and_then(Value::as_str)
+ {
+ if let Some(t) = types.get(f) {
+ local.insert(
+ f.to_string(),
+ t.clone(),
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+ if !local.is_empty() {
+ comp.insert(
+ "__xy_field_types__".to_string(),
+ Value::Object(local),
+ );
+ }
+ }
+ }
+ for c in obj.values_mut() {
+ annotate(c, types);
+ }
+ }
+ Value::Array(arr) => {
+ for item in arr {
+ annotate(item, types);
+ }
+ }
+ _ => {}
+ }
+ }
+ if let Some(aggs) = body.aggs.as_mut() {
+ annotate(aggs, &types);
+ }
+ if let Some(aggs) = body.aggregations.as_mut() {
+ annotate(aggs, &types);
+ }
+ }
+ }
+ }
+
// Inject the mapping's per-field date `format` into date_histogram /
// date_range / range agg bodies that don't carry an explicit
// `format`. ES uses the mapping's format to render bucket keys
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 13f4be851..939e10d23 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -5958,6 +5958,114 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {
// ── Composite aggregation ─────────────────────────────────────────────────────
+/// Key typing for a composite `terms` source, derived from the source
+/// field's MAPPING type. The HTTP layer injects the participating fields'
+/// mapping types into the composite params as `__xy_field_types__`
+/// (`{"field": "keyword", ...}`) — the same sentinel pattern
+/// `__xy_f32_fields__` uses for matrix_stats.
+///
+/// ES types composite keys from the field's `DocValueFormat`: keyword
+/// values stay strings (so `"007"` is NOT corrupted to the number 7 and
+/// keys sort lexicographically), booleans become JSON `true`/`false`,
+/// numeric fields become numbers. `Heuristic` (no mapping info available)
+/// preserves the legacy parse-based guess for unmapped fields.
+#[derive(Clone, Copy, PartialEq)]
+pub(crate) enum CompositeKeyKind {
+ Str,
+ Bool,
+ Int,
+ Float,
+ Heuristic,
+}
+
+pub(crate) fn composite_kind_of(mapping_type: Option<&str>) -> CompositeKeyKind {
+ match mapping_type {
+ Some("keyword") | Some("text") | Some("wildcard") | Some("constant_keyword")
+ | Some("match_only_text") | Some("ip") | Some("version") => CompositeKeyKind::Str,
+ Some("boolean") => CompositeKeyKind::Bool,
+ Some("long") | Some("integer") | Some("short") | Some("byte") | Some("unsigned_long") => {
+ CompositeKeyKind::Int
+ }
+ Some("double") | Some("float") | Some("half_float") | Some("scaled_float") => {
+ CompositeKeyKind::Float
+ }
+ _ => CompositeKeyKind::Heuristic,
+ }
+}
+
+/// Render one composite key part as its JSON value according to the
+/// source kind. Fallbacks never panic: a value that doesn't parse under
+/// its declared kind degrades to a string (never silently to a wrong
+/// number).
+pub(crate) fn composite_typed_key_value(kind: CompositeKeyKind, val: &str) -> Value {
+ match kind {
+ CompositeKeyKind::Str => Value::String(val.to_string()),
+ CompositeKeyKind::Bool => match val {
+ "true" => Value::Bool(true),
+ "false" => Value::Bool(false),
+ _ => Value::String(val.to_string()),
+ },
+ CompositeKeyKind::Float => {
+ // ES renders double keys with their fraction (2 → 2.0).
+ if let Ok(f) = val.parse::<f64>() {
+ serde_json::Number::from_f64(f)
+ .map(Value::Number)
+ .unwrap_or_else(|| Value::String(val.to_string()))
+ } else {
+ Value::String(val.to_string())
+ }
+ }
+ CompositeKeyKind::Int | CompositeKeyKind::Heuristic => {
+ if let Ok(n) = val.parse::<i64>() {
+ json!(n)
+ } else if let Ok(f) = val.parse::<f64>() {
+ serde_json::Number::from_f64(f)
+ .map(Value::Number)
+ .unwrap_or_else(|| Value::String(val.to_string()))
+ } else {
+ Value::String(val.to_string())
+ }
+ }
+ }
+}
+
+/// Compare two composite key parts under a source kind.
+/// `numeric_hint` is the legacy "source type is histogram/date_histogram"
+/// flag that forces numeric comparison in Heuristic mode.
+pub(crate) fn composite_part_cmp(
+ kind: CompositeKeyKind,
+ numeric_hint: bool,
+ av: &str,
+ bv: &str,
+) -> std::cmp::Ordering {
+ use std::cmp::Ordering;
+ let numeric_cmp = |av: &str, bv: &str| -> Option<Ordering> {
+ match (av.parse::<f64>(), bv.parse::<f64>()) {
+ (Ok(an), Ok(bn)) => an.partial_cmp(&bn),
+ _ => None,
+ }
+ };
+ match kind {
+ // ES sorts keyword terms by their UTF-8 bytes ("10" < "9") and
+ // booleans false < true ("false" < "true" lexicographically).
+ CompositeKeyKind::Str | CompositeKeyKind::Bool => av.cmp(bv),
+ CompositeKeyKind::Int | CompositeKeyKind::Float => {
+ numeric_cmp(av, bv).unwrap_or_else(|| av.cmp(bv))
+ }
+ CompositeKeyKind::Heuristic => {
+ if numeric_hint {
+ let an: f64 = av.parse().unwrap_or(0.0);
+ let bn: f64 = bv.parse().unwrap_or(0.0);
+ an.partial_cmp(&bn).unwrap_or(Ordering::Equal)
+ } else if let Some(o) = numeric_cmp(av, bv) {
+ o
+ } else {
+ av.cmp(bv)
+ }
+ }
+ }
+}
+
fn run_composite(
params: &Value,
sub_aggs: Option<&Value>,
@@ -6230,6 +6338,25 @@ fn run_composite(
.iter()
.map(|(_, src_type, _, _)| src_type == "geotile_grid")
.collect();
+ // Mapping-driven key typing for `terms` sources (item 7): the HTTP
+ // layer injects `__xy_field_types__` = {field → mapping type}. Sources
+ // without mapping info (or non-terms sources) stay on the legacy
+ // parse heuristic.
+ let field_types = params.get("__xy_field_types__").and_then(Value::as_object);
+ let kinds: Vec<CompositeKeyKind> = source_defs
+ .iter()
+ .map(|(_, src_type, field, _)| {
+ if src_type == "terms" {
+ composite_kind_of(
+ field_types
+ .and_then(|m| m.get(field))
+ .and_then(Value::as_str),
+ )
+ } else {
+ CompositeKeyKind::Heuristic
+ }
+ })
+ .collect();
/// Parse a geotile key `"z/x/y"` into a (z, x, y) tuple for
/// numeric sorting; returns `(0, 0, 0)` on parse error.
fn geotile_parse(s: &str) -> (i64, i64, i64) {
@@ -6259,14 +6386,13 @@ fn run_composite(
} else if geotile_src.get(i).copied().unwrap_or(false) {
// Geotile keys sort by (zoom, x, y) numerically.
geotile_parse(av).cmp(&geotile_parse(bv))
- } else if numeric_src.get(i).copied().unwrap_or(false)
- || (av.parse::<f64>().is_ok() && bv.parse::<f64>().is_ok())
- {
- let an: f64 = av.parse().unwrap_or(0.0);
- let bn: f64 = bv.parse().unwrap_or(0.0);
- an.partial_cmp(&bn).unwrap_or(Ordering::Equal)
} else {
- av.cmp(bv)
+ composite_part_cmp(
+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),
+ numeric_src.get(i).copied().unwrap_or(false),
+ av,
+ bv,
+ )
};
let cmp = if orders.get(i).copied().unwrap_or(true) {
cmp
@@ -6371,14 +6497,13 @@ fn run_composite(
Ordering::Greater
} else if geotile_src.get(i).copied().unwrap_or(false) {
geotile_parse(av).cmp(&geotile_parse(bv))
- } else if numeric_src.get(i).copied().unwrap_or(false)
- || (av.parse::<f64>().is_ok() && bv.parse::<f64>().is_ok())
- {
- let an: f64 = av.parse().unwrap_or(0.0);
- let bn: f64 = bv.parse().unwrap_or(0.0);
- an.partial_cmp(&bn).unwrap_or(Ordering::Equal)
} else {
- av.cmp(bv)
+ composite_part_cmp(
+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),
+ numeric_src.get(i).copied().unwrap_or(false),
+ av,
+ bv,
+ )
};
let c = if orders.get(i).copied().unwrap_or(true) {
c
@@ -6457,7 +6582,19 @@ fn run_composite(
} else {
key_obj.insert(src_name.clone(), Value::String(val));
}
- } else if src_type == "histogram" || src_type == "terms" {
+ } else if src_type == "terms" {
+ // Mapping-typed key (item 7): keyword stays a string
+ // ("007" is not corrupted to 7), boolean becomes JSON
+ // true/false, numeric mappings become numbers. Unmapped
+ // fields keep the legacy parse heuristic.
+ key_obj.insert(
+ src_name.clone(),
+ composite_typed_key_value(
+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),
+ &val,
+ ),
+ );
+ } else if src_type == "histogram" {
if let Ok(n) = val.parse::<i64>() {
key_obj.insert(src_name.clone(), json!(n));
} else if let Ok(f) = val.parse::<f64>() {
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 75913aa00..70486d982 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -3487,11 +3487,14 @@ impl<'a> FastCtx<'a> {
if sub.is_some() {
return None;
}
- if !params_only(params, &["sources", "size"]) {
+ // `__xy_field_types__` is the HTTP layer's mapping-type sentinel
+ // (item 7 key typing) — metadata, not a user param.
+ if !params_only(params, &["sources", "size", "__xy_field_types__"]) {
return None;
}
let sources = params.get("sources").and_then(Value::as_array)?;
let size = params.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;
+ let field_types = params.get("__xy_field_types__").and_then(Value::as_object);
// All sources must be plain `terms` over keyword columns.
let mut src_names: Vec<String> = Vec::new();
@@ -3521,6 +3524,16 @@ impl<'a> FastCtx<'a> {
if src_names.is_empty() {
return Some(json!({ "buckets": [] }));
}
+ // Mapping-driven key kinds, identical to run_composite's — keeps
+ // this arm byte-identical to brute for sorting AND key typing.
+ let kinds: Vec<crate::aggs::CompositeKeyKind> = src_fields
+ .iter()
+ .map(|f| {
+ crate::aggs::composite_kind_of(
+ field_types.and_then(|m| m.get(f)).and_then(Value::as_str),
+ )
+ })
+ .collect();
let mut counts: HashMap<Vec<String>, u64> = HashMap::new();
for seg in &self.segs {
@@ -3651,7 +3664,7 @@ impl<'a> FastCtx<'a> {
}
let mut sorted_keys: Vec<Vec<String>> = counts.keys().cloned().collect();
- sorted_keys.sort_by(|a, b| composite_cmp(a, b));
+ sorted_keys.sort_by(|a, b| composite_cmp(a, b, &kinds));
sorted_keys.truncate(size);
let result_buckets: Vec<Value> = sorted_keys
@@ -3660,18 +3673,14 @@ impl<'a> FastCtx<'a> {
let mut key_obj = Map::new();
for (i, name) in src_names.iter().enumerate() {
let val = key.get(i).cloned().unwrap_or_default();
- if let Ok(n) = val.parse::<i64>() {
- key_obj.insert(name.clone(), json!(n));
- } else if let Ok(f) = val.parse::<f64>() {
- key_obj.insert(
- name.clone(),
- serde_json::Number::from_f64(f)
- .map(Value::Number)
- .unwrap_or(Value::String(val.clone())),
- );
- } else {
- key_obj.insert(name.clone(), Value::String(val));
- }
+ let kind = kinds
+ .get(i)
+ .copied()
+ .unwrap_or(crate::aggs::CompositeKeyKind::Heuristic);
+ key_obj.insert(
+ name.clone(),
+ crate::aggs::composite_typed_key_value(kind, &val),
+ );
}
json!({ "key": Value::Object(key_obj), "doc_count": counts[key] })
})
@@ -4574,15 +4583,22 @@ fn cmp_by_orders(
/// Composite key comparison for terms-only sources (mirror of the brute
/// sort: numeric when both sides parse as f64, else lexicographic; asc).
-fn composite_cmp(a: &[String], b: &[String]) -> std::cmp::Ordering {
+/// Per-source composite key comparison — delegates each part to the
+/// mapping-driven `composite_part_cmp` so this fast arm sorts exactly
+/// like the brute `run_composite` (keyword sources lexicographic,
+/// numeric mappings numeric, unmapped → legacy parse heuristic).
+fn composite_cmp(
+ a: &[String],
+ b: &[String],
+ kinds: &[crate::aggs::CompositeKeyKind],
+) -> std::cmp::Ordering {
use std::cmp::Ordering;
for i in 0..a.len().min(b.len()) {
- let (av, bv) = (&a[i], &b[i]);
- let cmp = if let (Ok(an), Ok(bn)) = (av.parse::<f64>(), bv.parse::<f64>()) {
- an.partial_cmp(&bn).unwrap_or(Ordering::Equal)
- } else {
- av.cmp(bv)
- };
+ let kind = kinds
+ .get(i)
+ .copied()
+ .unwrap_or(crate::aggs::CompositeKeyKind::Heuristic);
+ let cmp = crate::aggs::composite_part_cmp(kind, false, &a[i], &b[i]);
if cmp != Ordering::Equal {
return cmp;
}