AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(es-compat): terms-agg keys typed by field source shape, not content (#864) (#867)
xerj-org608f30a59d01
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex b6e52901b..8670d47fe 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -3494,6 +3494,13 @@ fn run_terms(None => return json!({"buckets": []}),},};+ // #864: type bucket keys by the field's source shape — a keyword value that+ // looks numeric ("007") must stay a string. Scripts have no field mapping,+ // so keep the legacy content-based coercion for them.+ let source_is_string = match &field_or_script {+ FieldOrScript::Field(f) => field_source_is_string(docs, f),+ FieldOrScript::Script { .. } => false,+ };// ES semantics: size=0 means "return all buckets" (no cap).// Any other value caps the result; the default is 10.let size_opt: Option<usize> = params@@ -3777,7 +3784,7 @@ fn run_terms(let _ = &bucket_docs; // silence unused when neither sub nor `count` needs itlet (key, count, precomputed_sub) = (key, count, sub_res);- let (typed_key, key_as_string) = typed_term_key(&key);+ let (typed_key, key_as_string) = typed_term_key(&key, source_is_string);let mut bucket = json!({"key": typed_key,@@ -3874,9 +3881,52 @@ pub(crate) fn apply_terms_size_pipeline<T>(/// a string. `key_as_string` is only added when it carries extra/// information that the numeric key can't: booleans and dates.///+/// #864: `source_is_string` reflects whether the field's source values are+/// JSON STRINGS (keyword / text / date-family) rather than JSON numbers or+/// bools. ES types a bucket key by the field's MAPPING, so a keyword value+/// that merely LOOKS numeric (`"007"`) must stay the string `"007"`, not the+/// number 7 — the string branch skips i64/f64 coercion but still surfaces an+/// ISO date as epoch millis (a genuine `date` field's values are strings).+////// Shared by `run_terms` and the doc-values fast-agg path so bucket/// key shaping is byte-identical between the two.-pub(crate) fn typed_term_key(key: &str) -> (Value, Option<String>) {+/// #864: does this field hold JSON STRING values (keyword / text / date) rather+/// than JSON numbers or bools? Inspects the raw source of `docs` — the first+/// present scalar value's JSON type decides (ES types the whole field by its+/// mapping). No values / all null → `false` (there are no keys to shape anyway).+fn field_source_is_string(docs: &[Value], field: &str) -> bool {+ fn scalar_is_string(v: &Value) -> Option<bool> {+ match v {+ Value::String(_) => Some(true),+ Value::Number(_) | Value::Bool(_) => Some(false),+ Value::Array(a) => a.iter().find_map(scalar_is_string),+ _ => None,+ }+ }+ for doc in docs {+ if let Some(v) = crate::index::get_field_value(doc, field) {+ if let Some(is_str) = scalar_is_string(&v) {+ return is_str;+ }+ }+ }+ false+}++pub(crate) fn typed_term_key(key: &str, source_is_string: bool) -> (Value, Option<String>) {+ if source_is_string {+ // Keyword/text/date-family: a numeric-looking value ("007") stays a+ // string. A GENUINE ISO date string still surfaces as epoch millis — but+ // NOT a bare number (`parse_date_ms` treats "007" as epoch 7 via its+ // integer fallback, which would defeat the whole point).+ let is_bare_number = key.parse::<i64>().is_ok() || key.parse::<f64>().is_ok();+ if !is_bare_number {+ if let Some(epoch_ms) = parse_date_ms(&Value::String(key.to_string())) {+ return (json!(epoch_ms), Some(epoch_ms_to_iso8601_utc(epoch_ms)));+ }+ }+ return (json!(key), None);+ }let mut key_as_string: Option<String> = None;let typed_key: Value = if key == "true" || key == "false" {key_as_string = Some(key.to_string());diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex df4308674..d92280e41 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -3067,7 +3067,9 @@ impl<'a> FastCtx<'a> {let mut buckets: Vec<Value> = Vec::with_capacity(candidates.len());for (key, st) in candidates {- let (typed_key, key_as_string) = typed_term_key(&key);+ // #864: a keyword column's keys are strings (`is_bool` marks the+ // boolean-column case, which still renders true/false → 1/0).+ let (typed_key, key_as_string) = typed_term_key(&key, !is_bool);let mut bucket = Map::new();bucket.insert("key".to_string(), typed_key);if let Some(kas) = key_as_string {@@ -3178,7 +3180,9 @@ impl<'a> FastCtx<'a> {let buckets: Vec<Value> = entries.into_iter().map(|(key, count)| {- let (typed_key, key_as_string) = typed_term_key(&key);+ // #864: exec_rare_terms is keyword-only (guarded above), so the+ // keys are always strings.+ let (typed_key, key_as_string) = typed_term_key(&key, true);let mut b = Map::new();b.insert("key".to_string(), typed_key);if let Some(kas) = key_as_string {diff --git a/engine/crates/xerj-engine/tests/terms_key_typing.rs b/engine/crates/xerj-engine/tests/terms_key_typing.rsnew file mode 100644index 000000000..a5f963b61--- /dev/null+++ b/engine/crates/xerj-engine/tests/terms_key_typing.rs@@ -0,0 +1,80 @@+//! #864: a terms-agg bucket key is typed by the field's SOURCE shape. A keyword+//! value that looks numeric ("007") must stay the string "007" (ES keys a+//! keyword bucket by the string); a genuine numeric field still keys by number.+//! Asserted on the memtable (brute `run_terms`) AND after flush (columnar+//! `fast_aggs`) so the two paths agree.++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::Engine;+use xerj_query::parse_request;++async fn bucket_keys(engine: &Engine, field: &str) -> Vec<Value> {+ let idx = engine.get_index("docs").expect("get index");+ let req = parse_request(&json!({+ "size": 0,+ "aggs": { "by": { "terms": { "field": field } } }+ }))+ .expect("parse_request");+ let res = idx.search(&req).await.expect("search");+ res.aggs.expect("aggs")["by"]["buckets"]+ .as_array()+ .expect("buckets")+ .iter()+ .map(|b| b["key"].clone())+ .collect()+}++#[tokio::test]+async fn terms_keyword_key_stays_string_numeric_stays_number() {+ let dir = TempDir::new().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ let engine = Engine::new(config).expect("engine");++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("code", FieldType::Keyword));+ schema.fields.push(FieldConfig::new("qty", FieldType::Long));+ engine.create_index("docs", schema).expect("create");+ let idx = engine.get_index("docs").expect("get");+ for (id, code, qty) in [("d0", "007", 5), ("d1", "007", 5), ("d2", "042", 9)] {+ idx.index_document(Some(id.to_string()), json!({ "code": code, "qty": qty }))+ .await+ .expect("index");+ }++ // Assert buffered (memtable) and flushed (segment) agree AND are ES-correct.+ let kw_pre = bucket_keys(&engine, "code").await;+ let num_pre = bucket_keys(&engine, "qty").await;+ idx.refresh().await.expect("refresh");+ let kw_post = bucket_keys(&engine, "code").await;+ let num_post = bucket_keys(&engine, "qty").await;++ assert_eq!(+ kw_pre, kw_post,+ "#864: keyword key typing must be flush-invariant"+ );+ assert_eq!(+ num_pre, num_post,+ "numeric key typing must be flush-invariant"+ );++ // Keyword: string keys with leading zero intact.+ assert!(+ kw_post.contains(&json!("007")) && kw_post.contains(&json!("042")),+ "keyword keys must stay strings, got {kw_post:?}"+ );+ assert!(+ !kw_post.contains(&json!(7)) && !kw_post.contains(&json!(42)),+ "keyword keys must NOT be coerced to numbers, got {kw_post:?}"+ );+ // Numeric field: number keys (control — coercion still applies).+ assert!(+ num_post.contains(&json!(5)) && num_post.contains(&json!(9)),+ "numeric keys must stay numbers, got {num_post:?}"+ );+}