AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(aggs): type composite bucket keys from the source field MAPPING (RC4-W2 item 7)
xerj-org22b2bd292fa4
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 7254f6185..5819af4ab 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -5691,6 +5691,150 @@ pub async fn search(}}+ // Annotate every `composite` agg with `__xy_field_types__`: the mapping+ // type of each of its `terms` sources' fields. The engine types the+ // composite bucket keys from the SOURCE FIELD MAPPING (keyword → string,+ // boolean → true/false, numerics → numbers) instead of string-parse+ // heuristics that corrupted keyword values like "007" into the number 7+ // and rendered booleans as the strings "true"/"false".+ {+ fn gather_composite_terms_fields(v: &Value, out: &mut std::collections::HashSet<String>) {+ match v {+ Value::Object(obj) => {+ if let Some(sources) = obj+ .get("composite")+ .and_then(|c| c.get("sources"))+ .and_then(Value::as_array)+ {+ for src in sources {+ if let Some(src_obj) = src.as_object() {+ for spec in src_obj.values() {+ if let Some(f) = spec+ .get("terms")+ .and_then(|t| t.get("field"))+ .and_then(Value::as_str)+ {+ out.insert(f.to_string());+ }+ }+ }+ }+ }+ for c in obj.values() {+ gather_composite_terms_fields(c, out);+ }+ }+ Value::Array(arr) => {+ for item in arr {+ gather_composite_terms_fields(item, out);+ }+ }+ _ => {}+ }+ }+ let mut wanted: std::collections::HashSet<String> = std::collections::HashSet::new();+ if let Some(aggs) = body.aggs.as_ref() {+ gather_composite_terms_fields(aggs, &mut wanted);+ }+ if let Some(aggs) = body.aggregations.as_ref() {+ gather_composite_terms_fields(aggs, &mut wanted);+ }+ if !wanted.is_empty() {+ // Resolve a (possibly dotted / multi-field) path against a+ // mapping `properties` object: `user.name` walks properties,+ // `agent.keyword` walks multi-`fields`.+ fn type_in_props(props: &Value, field: &str) -> Option<String> {+ let mut cur = props;+ let mut parts = field.split('.').peekable();+ while let Some(part) = parts.next() {+ let spec = cur.get(part)?;+ if parts.peek().is_none() {+ return spec.get("type").and_then(Value::as_str).map(str::to_string);+ }+ cur = spec.get("properties").or_else(|| spec.get("fields"))?;+ }+ None+ }+ let mut types: serde_json::Map<String, Value> = serde_json::Map::new();+ for f in &wanted {+ for ix in &index_names {+ if let Some(m) = state.engine.index_mappings.get(*ix) {+ let mapping = m.clone();+ let props = mapping+ .get("mappings")+ .and_then(|mm| mm.get("properties"))+ .or_else(|| mapping.get("properties"))+ .cloned();+ if let Some(p) = props {+ if let Some(t) = type_in_props(&p, f) {+ types.insert(f.clone(), Value::String(t));+ break;+ }+ }+ }+ }+ }+ if !types.is_empty() {+ fn annotate(v: &mut Value, types: &serde_json::Map<String, Value>) {+ match v {+ Value::Object(obj) => {+ if let Some(comp) =+ obj.get_mut("composite").and_then(|c| c.as_object_mut())+ {+ if !comp.contains_key("__xy_field_types__") {+ let mut local = serde_json::Map::new();+ if let Some(sources) =+ comp.get("sources").and_then(Value::as_array)+ {+ for src in sources {+ if let Some(src_obj) = src.as_object() {+ for spec in src_obj.values() {+ if let Some(f) = spec+ .get("terms")+ .and_then(|t| t.get("field"))+ .and_then(Value::as_str)+ {+ if let Some(t) = types.get(f) {+ local.insert(+ f.to_string(),+ t.clone(),+ );+ }+ }+ }+ }+ }+ }+ if !local.is_empty() {+ comp.insert(+ "__xy_field_types__".to_string(),+ Value::Object(local),+ );+ }+ }+ }+ for c in obj.values_mut() {+ annotate(c, types);+ }+ }+ Value::Array(arr) => {+ for item in arr {+ annotate(item, types);+ }+ }+ _ => {}+ }+ }+ if let Some(aggs) = body.aggs.as_mut() {+ annotate(aggs, &types);+ }+ if let Some(aggs) = body.aggregations.as_mut() {+ annotate(aggs, &types);+ }+ }+ }+ }+// Inject the mapping's per-field date `format` into date_histogram /// date_range / range agg bodies that don't carry an explicit// `format`. ES uses the mapping's format to render bucket keysdiff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 13f4be851..939e10d23 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -5958,6 +5958,114 @@ fn run_percentiles(params: &Value, docs: &[Value]) -> Value {// ── Composite aggregation ─────────────────────────────────────────────────────+/// Key typing for a composite `terms` source, derived from the source+/// field's MAPPING type. The HTTP layer injects the participating fields'+/// mapping types into the composite params as `__xy_field_types__`+/// (`{"field": "keyword", ...}`) — the same sentinel pattern+/// `__xy_f32_fields__` uses for matrix_stats.+///+/// ES types composite keys from the field's `DocValueFormat`: keyword+/// values stay strings (so `"007"` is NOT corrupted to the number 7 and+/// keys sort lexicographically), booleans become JSON `true`/`false`,+/// numeric fields become numbers. `Heuristic` (no mapping info available)+/// preserves the legacy parse-based guess for unmapped fields.+#[derive(Clone, Copy, PartialEq)]+pub(crate) enum CompositeKeyKind {+ Str,+ Bool,+ Int,+ Float,+ Heuristic,+}++pub(crate) fn composite_kind_of(mapping_type: Option<&str>) -> CompositeKeyKind {+ match mapping_type {+ Some("keyword") | Some("text") | Some("wildcard") | Some("constant_keyword")+ | Some("match_only_text") | Some("ip") | Some("version") => CompositeKeyKind::Str,+ Some("boolean") => CompositeKeyKind::Bool,+ Some("long") | Some("integer") | Some("short") | Some("byte") | Some("unsigned_long") => {+ CompositeKeyKind::Int+ }+ Some("double") | Some("float") | Some("half_float") | Some("scaled_float") => {+ CompositeKeyKind::Float+ }+ _ => CompositeKeyKind::Heuristic,+ }+}++/// Render one composite key part as its JSON value according to the+/// source kind. Fallbacks never panic: a value that doesn't parse under+/// its declared kind degrades to a string (never silently to a wrong+/// number).+pub(crate) fn composite_typed_key_value(kind: CompositeKeyKind, val: &str) -> Value {+ match kind {+ CompositeKeyKind::Str => Value::String(val.to_string()),+ CompositeKeyKind::Bool => match val {+ "true" => Value::Bool(true),+ "false" => Value::Bool(false),+ _ => Value::String(val.to_string()),+ },+ CompositeKeyKind::Float => {+ // ES renders double keys with their fraction (2 → 2.0).+ if let Ok(f) = val.parse::<f64>() {+ serde_json::Number::from_f64(f)+ .map(Value::Number)+ .unwrap_or_else(|| Value::String(val.to_string()))+ } else {+ Value::String(val.to_string())+ }+ }+ CompositeKeyKind::Int | CompositeKeyKind::Heuristic => {+ if let Ok(n) = val.parse::<i64>() {+ json!(n)+ } else if let Ok(f) = val.parse::<f64>() {+ serde_json::Number::from_f64(f)+ .map(Value::Number)+ .unwrap_or_else(|| Value::String(val.to_string()))+ } else {+ Value::String(val.to_string())+ }+ }+ }+}++/// Compare two composite key parts under a source kind.+/// `numeric_hint` is the legacy "source type is histogram/date_histogram"+/// flag that forces numeric comparison in Heuristic mode.+pub(crate) fn composite_part_cmp(+ kind: CompositeKeyKind,+ numeric_hint: bool,+ av: &str,+ bv: &str,+) -> std::cmp::Ordering {+ use std::cmp::Ordering;+ let numeric_cmp = |av: &str, bv: &str| -> Option<Ordering> {+ match (av.parse::<f64>(), bv.parse::<f64>()) {+ (Ok(an), Ok(bn)) => an.partial_cmp(&bn),+ _ => None,+ }+ };+ match kind {+ // ES sorts keyword terms by their UTF-8 bytes ("10" < "9") and+ // booleans false < true ("false" < "true" lexicographically).+ CompositeKeyKind::Str | CompositeKeyKind::Bool => av.cmp(bv),+ CompositeKeyKind::Int | CompositeKeyKind::Float => {+ numeric_cmp(av, bv).unwrap_or_else(|| av.cmp(bv))+ }+ CompositeKeyKind::Heuristic => {+ if numeric_hint {+ let an: f64 = av.parse().unwrap_or(0.0);+ let bn: f64 = bv.parse().unwrap_or(0.0);+ an.partial_cmp(&bn).unwrap_or(Ordering::Equal)+ } else if let Some(o) = numeric_cmp(av, bv) {+ o+ } else {+ av.cmp(bv)+ }+ }+ }+}+fn run_composite(params: &Value,sub_aggs: Option<&Value>,@@ -6230,6 +6338,25 @@ fn run_composite(.iter().map(|(_, src_type, _, _)| src_type == "geotile_grid").collect();+ // Mapping-driven key typing for `terms` sources (item 7): the HTTP+ // layer injects `__xy_field_types__` = {field → mapping type}. Sources+ // without mapping info (or non-terms sources) stay on the legacy+ // parse heuristic.+ let field_types = params.get("__xy_field_types__").and_then(Value::as_object);+ let kinds: Vec<CompositeKeyKind> = source_defs+ .iter()+ .map(|(_, src_type, field, _)| {+ if src_type == "terms" {+ composite_kind_of(+ field_types+ .and_then(|m| m.get(field))+ .and_then(Value::as_str),+ )+ } else {+ CompositeKeyKind::Heuristic+ }+ })+ .collect();/// Parse a geotile key `"z/x/y"` into a (z, x, y) tuple for/// numeric sorting; returns `(0, 0, 0)` on parse error.fn geotile_parse(s: &str) -> (i64, i64, i64) {@@ -6259,14 +6386,13 @@ fn run_composite(} else if geotile_src.get(i).copied().unwrap_or(false) {// Geotile keys sort by (zoom, x, y) numerically.geotile_parse(av).cmp(&geotile_parse(bv))- } else if numeric_src.get(i).copied().unwrap_or(false)- || (av.parse::<f64>().is_ok() && bv.parse::<f64>().is_ok())- {- let an: f64 = av.parse().unwrap_or(0.0);- let bn: f64 = bv.parse().unwrap_or(0.0);- an.partial_cmp(&bn).unwrap_or(Ordering::Equal)} else {- av.cmp(bv)+ composite_part_cmp(+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),+ numeric_src.get(i).copied().unwrap_or(false),+ av,+ bv,+ )};let cmp = if orders.get(i).copied().unwrap_or(true) {cmp@@ -6371,14 +6497,13 @@ fn run_composite(Ordering::Greater} else if geotile_src.get(i).copied().unwrap_or(false) {geotile_parse(av).cmp(&geotile_parse(bv))- } else if numeric_src.get(i).copied().unwrap_or(false)- || (av.parse::<f64>().is_ok() && bv.parse::<f64>().is_ok())- {- let an: f64 = av.parse().unwrap_or(0.0);- let bn: f64 = bv.parse().unwrap_or(0.0);- an.partial_cmp(&bn).unwrap_or(Ordering::Equal)} else {- av.cmp(bv)+ composite_part_cmp(+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),+ numeric_src.get(i).copied().unwrap_or(false),+ av,+ bv,+ )};let c = if orders.get(i).copied().unwrap_or(true) {c@@ -6457,7 +6582,19 @@ fn run_composite(} else {key_obj.insert(src_name.clone(), Value::String(val));}- } else if src_type == "histogram" || src_type == "terms" {+ } else if src_type == "terms" {+ // Mapping-typed key (item 7): keyword stays a string+ // ("007" is not corrupted to 7), boolean becomes JSON+ // true/false, numeric mappings become numbers. Unmapped+ // fields keep the legacy parse heuristic.+ key_obj.insert(+ src_name.clone(),+ composite_typed_key_value(+ kinds.get(i).copied().unwrap_or(CompositeKeyKind::Heuristic),+ &val,+ ),+ );+ } else if src_type == "histogram" {if let Ok(n) = val.parse::<i64>() {key_obj.insert(src_name.clone(), json!(n));} else if let Ok(f) = val.parse::<f64>() {diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 75913aa00..70486d982 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -3487,11 +3487,14 @@ impl<'a> FastCtx<'a> {if sub.is_some() {return None;}- if !params_only(params, &["sources", "size"]) {+ // `__xy_field_types__` is the HTTP layer's mapping-type sentinel+ // (item 7 key typing) — metadata, not a user param.+ if !params_only(params, &["sources", "size", "__xy_field_types__"]) {return None;}let sources = params.get("sources").and_then(Value::as_array)?;let size = params.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;+ let field_types = params.get("__xy_field_types__").and_then(Value::as_object);// All sources must be plain `terms` over keyword columns.let mut src_names: Vec<String> = Vec::new();@@ -3521,6 +3524,16 @@ impl<'a> FastCtx<'a> {if src_names.is_empty() {return Some(json!({ "buckets": [] }));}+ // Mapping-driven key kinds, identical to run_composite's — keeps+ // this arm byte-identical to brute for sorting AND key typing.+ let kinds: Vec<crate::aggs::CompositeKeyKind> = src_fields+ .iter()+ .map(|f| {+ crate::aggs::composite_kind_of(+ field_types.and_then(|m| m.get(f)).and_then(Value::as_str),+ )+ })+ .collect();let mut counts: HashMap<Vec<String>, u64> = HashMap::new();for seg in &self.segs {@@ -3651,7 +3664,7 @@ impl<'a> FastCtx<'a> {}let mut sorted_keys: Vec<Vec<String>> = counts.keys().cloned().collect();- sorted_keys.sort_by(|a, b| composite_cmp(a, b));+ sorted_keys.sort_by(|a, b| composite_cmp(a, b, &kinds));sorted_keys.truncate(size);let result_buckets: Vec<Value> = sorted_keys@@ -3660,18 +3673,14 @@ impl<'a> FastCtx<'a> {let mut key_obj = Map::new();for (i, name) in src_names.iter().enumerate() {let val = key.get(i).cloned().unwrap_or_default();- if let Ok(n) = val.parse::<i64>() {- key_obj.insert(name.clone(), json!(n));- } else if let Ok(f) = val.parse::<f64>() {- key_obj.insert(- name.clone(),- serde_json::Number::from_f64(f)- .map(Value::Number)- .unwrap_or(Value::String(val.clone())),- );- } else {- key_obj.insert(name.clone(), Value::String(val));- }+ let kind = kinds+ .get(i)+ .copied()+ .unwrap_or(crate::aggs::CompositeKeyKind::Heuristic);+ key_obj.insert(+ name.clone(),+ crate::aggs::composite_typed_key_value(kind, &val),+ );}json!({ "key": Value::Object(key_obj), "doc_count": counts[key] })})@@ -4574,15 +4583,22 @@ fn cmp_by_orders(/// Composite key comparison for terms-only sources (mirror of the brute/// sort: numeric when both sides parse as f64, else lexicographic; asc).-fn composite_cmp(a: &[String], b: &[String]) -> std::cmp::Ordering {+/// Per-source composite key comparison — delegates each part to the+/// mapping-driven `composite_part_cmp` so this fast arm sorts exactly+/// like the brute `run_composite` (keyword sources lexicographic,+/// numeric mappings numeric, unmapped → legacy parse heuristic).+fn composite_cmp(+ a: &[String],+ b: &[String],+ kinds: &[crate::aggs::CompositeKeyKind],+) -> std::cmp::Ordering {use std::cmp::Ordering;for i in 0..a.len().min(b.len()) {- let (av, bv) = (&a[i], &b[i]);- let cmp = if let (Ok(an), Ok(bn)) = (av.parse::<f64>(), bv.parse::<f64>()) {- an.partial_cmp(&bn).unwrap_or(Ordering::Equal)- } else {- av.cmp(bv)- };+ let kind = kinds+ .get(i)+ .copied()+ .unwrap_or(crate::aggs::CompositeKeyKind::Heuristic);+ let cmp = crate::aggs::composite_part_cmp(kind, false, &a[i], &b[i]);if cmp != Ordering::Equal {return cmp;}