Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(es-compat): terms-agg keys typed by field source shape, not content (#864) (#867)

xerj-org608f30a59d01
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index b6e52901b..8670d47fe 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -3494,6 +3494,13 @@ fn run_terms(
None => return json!({"buckets": []}),
},
};
+ // #864: type bucket keys by the field's source shape — a keyword value that
+ // looks numeric ("007") must stay a string. Scripts have no field mapping,
+ // so keep the legacy content-based coercion for them.
+ let source_is_string = match &field_or_script {
+ FieldOrScript::Field(f) => field_source_is_string(docs, f),
+ FieldOrScript::Script { .. } => false,
+ };
// ES semantics: size=0 means "return all buckets" (no cap).
// Any other value caps the result; the default is 10.
let size_opt: Option<usize> = params
@@ -3777,7 +3784,7 @@ fn run_terms(
let _ = &bucket_docs; // silence unused when neither sub nor `count` needs it
let (key, count, precomputed_sub) = (key, count, sub_res);
- let (typed_key, key_as_string) = typed_term_key(&key);
+ let (typed_key, key_as_string) = typed_term_key(&key, source_is_string);
let mut bucket = json!({
"key": typed_key,
@@ -3874,9 +3881,52 @@ pub(crate) fn apply_terms_size_pipeline<T>(
/// a string. `key_as_string` is only added when it carries extra
/// information that the numeric key can't: booleans and dates.
///
+/// #864: `source_is_string` reflects whether the field's source values are
+/// JSON STRINGS (keyword / text / date-family) rather than JSON numbers or
+/// bools. ES types a bucket key by the field's MAPPING, so a keyword value
+/// that merely LOOKS numeric (`"007"`) must stay the string `"007"`, not the
+/// number 7 — the string branch skips i64/f64 coercion but still surfaces an
+/// ISO date as epoch millis (a genuine `date` field's values are strings).
+///
/// Shared by `run_terms` and the doc-values fast-agg path so bucket
/// key shaping is byte-identical between the two.
-pub(crate) fn typed_term_key(key: &str) -> (Value, Option<String>) {
+/// #864: does this field hold JSON STRING values (keyword / text / date) rather
+/// than JSON numbers or bools? Inspects the raw source of `docs` — the first
+/// present scalar value's JSON type decides (ES types the whole field by its
+/// mapping). No values / all null → `false` (there are no keys to shape anyway).
+fn field_source_is_string(docs: &[Value], field: &str) -> bool {
+ fn scalar_is_string(v: &Value) -> Option<bool> {
+ match v {
+ Value::String(_) => Some(true),
+ Value::Number(_) | Value::Bool(_) => Some(false),
+ Value::Array(a) => a.iter().find_map(scalar_is_string),
+ _ => None,
+ }
+ }
+ for doc in docs {
+ if let Some(v) = crate::index::get_field_value(doc, field) {
+ if let Some(is_str) = scalar_is_string(&v) {
+ return is_str;
+ }
+ }
+ }
+ false
+}
+
+pub(crate) fn typed_term_key(key: &str, source_is_string: bool) -> (Value, Option<String>) {
+ if source_is_string {
+ // Keyword/text/date-family: a numeric-looking value ("007") stays a
+ // string. A GENUINE ISO date string still surfaces as epoch millis — but
+ // NOT a bare number (`parse_date_ms` treats "007" as epoch 7 via its
+ // integer fallback, which would defeat the whole point).
+ let is_bare_number = key.parse::<i64>().is_ok() || key.parse::<f64>().is_ok();
+ if !is_bare_number {
+ if let Some(epoch_ms) = parse_date_ms(&Value::String(key.to_string())) {
+ return (json!(epoch_ms), Some(epoch_ms_to_iso8601_utc(epoch_ms)));
+ }
+ }
+ return (json!(key), None);
+ }
let mut key_as_string: Option<String> = None;
let typed_key: Value = if key == "true" || key == "false" {
key_as_string = Some(key.to_string());
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index df4308674..d92280e41 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -3067,7 +3067,9 @@ impl<'a> FastCtx<'a> {
let mut buckets: Vec<Value> = Vec::with_capacity(candidates.len());
for (key, st) in candidates {
- let (typed_key, key_as_string) = typed_term_key(&key);
+ // #864: a keyword column's keys are strings (`is_bool` marks the
+ // boolean-column case, which still renders true/false → 1/0).
+ let (typed_key, key_as_string) = typed_term_key(&key, !is_bool);
let mut bucket = Map::new();
bucket.insert("key".to_string(), typed_key);
if let Some(kas) = key_as_string {
@@ -3178,7 +3180,9 @@ impl<'a> FastCtx<'a> {
let buckets: Vec<Value> = entries
.into_iter()
.map(|(key, count)| {
- let (typed_key, key_as_string) = typed_term_key(&key);
+ // #864: exec_rare_terms is keyword-only (guarded above), so the
+ // keys are always strings.
+ let (typed_key, key_as_string) = typed_term_key(&key, true);
let mut b = Map::new();
b.insert("key".to_string(), typed_key);
if let Some(kas) = key_as_string {
diff --git a/engine/crates/xerj-engine/tests/terms_key_typing.rs b/engine/crates/xerj-engine/tests/terms_key_typing.rs
new file mode 100644
index 000000000..a5f963b61
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/terms_key_typing.rs
@@ -0,0 +1,80 @@
+//! #864: a terms-agg bucket key is typed by the field's SOURCE shape. A keyword
+//! value that looks numeric ("007") must stay the string "007" (ES keys a
+//! keyword bucket by the string); a genuine numeric field still keys by number.
+//! Asserted on the memtable (brute `run_terms`) AND after flush (columnar
+//! `fast_aggs`) so the two paths agree.
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+async fn bucket_keys(engine: &Engine, field: &str) -> Vec<Value> {
+ let idx = engine.get_index("docs").expect("get index");
+ let req = parse_request(&json!({
+ "size": 0,
+ "aggs": { "by": { "terms": { "field": field } } }
+ }))
+ .expect("parse_request");
+ let res = idx.search(&req).await.expect("search");
+ res.aggs.expect("aggs")["by"]["buckets"]
+ .as_array()
+ .expect("buckets")
+ .iter()
+ .map(|b| b["key"].clone())
+ .collect()
+}
+
+#[tokio::test]
+async fn terms_keyword_key_stays_string_numeric_stays_number() {
+ let dir = TempDir::new().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ let engine = Engine::new(config).expect("engine");
+
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("code", FieldType::Keyword));
+ schema.fields.push(FieldConfig::new("qty", FieldType::Long));
+ engine.create_index("docs", schema).expect("create");
+ let idx = engine.get_index("docs").expect("get");
+ for (id, code, qty) in [("d0", "007", 5), ("d1", "007", 5), ("d2", "042", 9)] {
+ idx.index_document(Some(id.to_string()), json!({ "code": code, "qty": qty }))
+ .await
+ .expect("index");
+ }
+
+ // Assert buffered (memtable) and flushed (segment) agree AND are ES-correct.
+ let kw_pre = bucket_keys(&engine, "code").await;
+ let num_pre = bucket_keys(&engine, "qty").await;
+ idx.refresh().await.expect("refresh");
+ let kw_post = bucket_keys(&engine, "code").await;
+ let num_post = bucket_keys(&engine, "qty").await;
+
+ assert_eq!(
+ kw_pre, kw_post,
+ "#864: keyword key typing must be flush-invariant"
+ );
+ assert_eq!(
+ num_pre, num_post,
+ "numeric key typing must be flush-invariant"
+ );
+
+ // Keyword: string keys with leading zero intact.
+ assert!(
+ kw_post.contains(&json!("007")) && kw_post.contains(&json!("042")),
+ "keyword keys must stay strings, got {kw_post:?}"
+ );
+ assert!(
+ !kw_post.contains(&json!(7)) && !kw_post.contains(&json!(42)),
+ "keyword keys must NOT be coerced to numbers, got {kw_post:?}"
+ );
+ // Numeric field: number keys (control — coercion still applies).
+ assert!(
+ num_post.contains(&json!(5)) && num_post.contains(&json!(9)),
+ "numeric keys must stay numbers, got {num_post:?}"
+ );
+}